diff --git a/WebCrawler/.vscode/launch.json b/WebCrawler/.vscode/launch.json new file mode 100644 index 0000000..7eb6a83 --- /dev/null +++ b/WebCrawler/.vscode/launch.json @@ -0,0 +1,18 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + + { + "type": "pwa-node", + "request": "launch", + "name": "Launch Program", + "skipFiles": [ + "/**" + ], + "program": "${workspaceFolder}\\postscrape\\postscrape\\spiders\\queries.js" + } + ] +} \ No newline at end of file diff --git a/WebCrawler/postscrape/itens.json b/WebCrawler/postscrape/itens.json new file mode 100644 index 0000000..276c35a --- /dev/null +++ b/WebCrawler/postscrape/itens.json @@ -0,0 +1,102 @@ +[ +{"text": "\u201cThe world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.\u201d", "author": {"name": "Albert Einstein", "url": "http://quotes.toscrape.com/author/Albert-Einstein"}, "tag": ["change", "deep-thoughts", "thinking", "world"]}, +{"text": "\u201cIt is our choices, Harry, that show what we truly are, far more than our abilities.\u201d", "author": {"name": "J.K. Rowling", "url": "http://quotes.toscrape.com/author/J-K-Rowling"}, "tag": ["abilities", "choices"]}, +{"text": "\u201cThere are only two ways to live your life. One is as though nothing is a miracle. The other is as though everything is a miracle.\u201d", "author": {"name": "Albert Einstein", "url": "http://quotes.toscrape.com/author/Albert-Einstein"}, "tag": ["inspirational", "life", "live", "miracle", "miracles"]}, +{"text": "\u201cThe person, be it gentleman or lady, who has not pleasure in a good novel, must be intolerably stupid.\u201d", "author": {"name": "Jane Austen", "url": "http://quotes.toscrape.com/author/Jane-Austen"}, "tag": ["aliteracy", "books", "classic", "humor"]}, +{"text": "\u201cImperfection is beauty, madness is genius and it's better to be absolutely ridiculous than absolutely boring.\u201d", "author": {"name": "Marilyn Monroe", "url": "http://quotes.toscrape.com/author/Marilyn-Monroe"}, "tag": ["be-yourself", "inspirational"]}, +{"text": "\u201cTry not to become a man of success. Rather become a man of value.\u201d", "author": {"name": "Albert Einstein", "url": "http://quotes.toscrape.com/author/Albert-Einstein"}, "tag": ["adulthood", "success", "value"]}, +{"text": "\u201cIt is better to be hated for what you are than to be loved for what you are not.\u201d", "author": {"name": "Andr\u00e9 Gide", "url": "http://quotes.toscrape.com/author/Andre-Gide"}, "tag": ["life", "love"]}, +{"text": "\u201cI have not failed. I've just found 10,000 ways that won't work.\u201d", "author": {"name": "Thomas A. Edison", "url": "http://quotes.toscrape.com/author/Thomas-A-Edison"}, "tag": ["edison", "failure", "inspirational", "paraphrased"]}, +{"text": "\u201cA woman is like a tea bag; you never know how strong it is until it's in hot water.\u201d", "author": {"name": "Eleanor Roosevelt", "url": "http://quotes.toscrape.com/author/Eleanor-Roosevelt"}, "tag": ["misattributed-eleanor-roosevelt"]}, +{"text": "\u201cA day without sunshine is like, you know, night.\u201d", "author": {"name": "Steve Martin", "url": "http://quotes.toscrape.com/author/Steve-Martin"}, "tag": ["humor", "obvious", "simile"]}, +{"text": "\u201cThis life is what you make it. No matter what, you're going to mess up sometimes, it's a universal truth. But the good part is you get to decide how you're going to mess it up. Girls will be your friends - they'll act like it anyway. But just remember, some come, some go. The ones that stay with you through everything - they're your true best friends. Don't let go of them. Also remember, sisters make the best friends in the world. As for lovers, well, they'll come and go too. And baby, I hate to say it, most of them - actually pretty much all of them are going to break your heart, but you can't give up because if you give up, you'll never find your soulmate. You'll never find that half who makes you whole and that goes for everything. Just because you fail once, doesn't mean you're gonna fail at everything. Keep trying, hold on, and always, always, always believe in yourself, because if you don't, then who will, sweetie? So keep your head high, keep your chin up, and most importantly, keep smiling, because life's a beautiful thing and there's so much to smile about.\u201d", "author": {"name": "Marilyn Monroe", "url": "http://quotes.toscrape.com/author/Marilyn-Monroe"}, "tag": ["friends", "heartbreak", "inspirational", "life", "love", "sisters"]}, +{"text": "\u201cIt takes a great deal of bravery to stand up to our enemies, but just as much to stand up to our friends.\u201d", "author": {"name": "J.K. Rowling", "url": "http://quotes.toscrape.com/author/J-K-Rowling"}, "tag": ["courage", "friends"]}, +{"text": "\u201cIf you can't explain it to a six year old, you don't understand it yourself.\u201d", "author": {"name": "Albert Einstein", "url": "http://quotes.toscrape.com/author/Albert-Einstein"}, "tag": ["simplicity", "understand"]}, +{"text": "\u201cYou may not be her first, her last, or her only. She loved before she may love again. But if she loves you now, what else matters? She's not perfect\u2014you aren't either, and the two of you may never be perfect together but if she can make you laugh, cause you to think twice, and admit to being human and making mistakes, hold onto her and give her the most you can. She may not be thinking about you every second of the day, but she will give you a part of her that she knows you can break\u2014her heart. So don't hurt her, don't change her, don't analyze and don't expect more than she can give. Smile when she makes you happy, let her know when she makes you mad, and miss her when she's not there.\u201d", "author": {"name": "Bob Marley", "url": "http://quotes.toscrape.com/author/Bob-Marley"}, "tag": ["love"]}, +{"text": "\u201cI like nonsense, it wakes up the brain cells. Fantasy is a necessary ingredient in living.\u201d", "author": {"name": "Dr. Seuss", "url": "http://quotes.toscrape.com/author/Dr-Seuss"}, "tag": ["fantasy"]}, +{"text": "\u201cI may not have gone where I intended to go, but I think I have ended up where I needed to be.\u201d", "author": {"name": "Douglas Adams", "url": "http://quotes.toscrape.com/author/Douglas-Adams"}, "tag": ["life", "navigation"]}, +{"text": "\u201cThe opposite of love is not hate, it's indifference. The opposite of art is not ugliness, it's indifference. The opposite of faith is not heresy, it's indifference. And the opposite of life is not death, it's indifference.\u201d", "author": {"name": "Elie Wiesel", "url": "http://quotes.toscrape.com/author/Elie-Wiesel"}, "tag": ["activism", "apathy", "hate", "indifference", "inspirational", "love", "opposite", "philosophy"]}, +{"text": "\u201cIt is not a lack of love, but a lack of friendship that makes unhappy marriages.\u201d", "author": {"name": "Friedrich Nietzsche", "url": "http://quotes.toscrape.com/author/Friedrich-Nietzsche"}, "tag": ["friendship", "lack-of-friendship", "lack-of-love", "love", "marriage", "unhappy-marriage"]}, +{"text": "\u201cGood friends, good books, and a sleepy conscience: this is the ideal life.\u201d", "author": {"name": "Mark Twain", "url": "http://quotes.toscrape.com/author/Mark-Twain"}, "tag": ["books", "contentment", "friends", "friendship", "life"]}, +{"text": "\u201cLife is what happens to us while we are making other plans.\u201d", "author": {"name": "Allen Saunders", "url": "http://quotes.toscrape.com/author/Allen-Saunders"}, "tag": ["fate", "life", "misattributed-john-lennon", "planning", "plans"]}, +{"text": "\u201cI love you without knowing how, or when, or from where. I love you simply, without problems or pride: I love you in this way because I do not know any other way of loving but this, in which there is no I or you, so intimate that your hand upon my chest is my hand, so intimate that when I fall asleep your eyes close.\u201d", "author": {"name": "Pablo Neruda", "url": "http://quotes.toscrape.com/author/Pablo-Neruda"}, "tag": ["love", "poetry"]}, +{"text": "\u201cFor every minute you are angry you lose sixty seconds of happiness.\u201d", "author": {"name": "Ralph Waldo Emerson", "url": "http://quotes.toscrape.com/author/Ralph-Waldo-Emerson"}, "tag": ["happiness"]}, +{"text": "\u201cIf you judge people, you have no time to love them.\u201d", "author": {"name": "Mother Teresa", "url": "http://quotes.toscrape.com/author/Mother-Teresa"}, "tag": ["attributed-no-source"]}, +{"text": "\u201cAnyone who thinks sitting in church can make you a Christian must also think that sitting in a garage can make you a car.\u201d", "author": {"name": "Garrison Keillor", "url": "http://quotes.toscrape.com/author/Garrison-Keillor"}, "tag": ["humor", "religion"]}, +{"text": "\u201cBeauty is in the eye of the beholder and it may be necessary from time to time to give a stupid or misinformed beholder a black eye.\u201d", "author": {"name": "Jim Henson", "url": "http://quotes.toscrape.com/author/Jim-Henson"}, "tag": ["humor"]}, +{"text": "\u201cToday you are You, that is truer than true. There is no one alive who is Youer than You.\u201d", "author": {"name": "Dr. Seuss", "url": "http://quotes.toscrape.com/author/Dr-Seuss"}, "tag": ["comedy", "life", "yourself"]}, +{"text": "\u201cIf you want your children to be intelligent, read them fairy tales. If you want them to be more intelligent, read them more fairy tales.\u201d", "author": {"name": "Albert Einstein", "url": "http://quotes.toscrape.com/author/Albert-Einstein"}, "tag": ["children", "fairy-tales"]}, +{"text": "\u201cIt is impossible to live without failing at something, unless you live so cautiously that you might as well not have lived at all - in which case, you fail by default.\u201d", "author": {"name": "J.K. Rowling", "url": "http://quotes.toscrape.com/author/J-K-Rowling"}, "tag": []}, +{"text": "\u201cLogic will get you from A to Z; imagination will get you everywhere.\u201d", "author": {"name": "Albert Einstein", "url": "http://quotes.toscrape.com/author/Albert-Einstein"}, "tag": ["imagination"]}, +{"text": "\u201cOne good thing about music, when it hits you, you feel no pain.\u201d", "author": {"name": "Bob Marley", "url": "http://quotes.toscrape.com/author/Bob-Marley"}, "tag": ["music"]}, +{"text": "\u201cThe more that you read, the more things you will know. The more that you learn, the more places you'll go.\u201d", "author": {"name": "Dr. Seuss", "url": "http://quotes.toscrape.com/author/Dr-Seuss"}, "tag": ["learning", "reading", "seuss"]}, +{"text": "\u201cOf course it is happening inside your head, Harry, but why on earth should that mean that it is not real?\u201d", "author": {"name": "J.K. Rowling", "url": "http://quotes.toscrape.com/author/J-K-Rowling"}, "tag": ["dumbledore"]}, +{"text": "\u201cThe truth is, everyone is going to hurt you. You just got to find the ones worth suffering for.\u201d", "author": {"name": "Bob Marley", "url": "http://quotes.toscrape.com/author/Bob-Marley"}, "tag": ["friendship"]}, +{"text": "\u201cNot all of us can do great things. But we can do small things with great love.\u201d", "author": {"name": "Mother Teresa", "url": "http://quotes.toscrape.com/author/Mother-Teresa"}, "tag": ["misattributed-to-mother-teresa", "paraphrased"]}, +{"text": "\u201cTo the well-organized mind, death is but the next great adventure.\u201d", "author": {"name": "J.K. Rowling", "url": "http://quotes.toscrape.com/author/J-K-Rowling"}, "tag": ["death", "inspirational"]}, +{"text": "\u201cAll you need is love. But a little chocolate now and then doesn't hurt.\u201d", "author": {"name": "Charles M. Schulz", "url": "http://quotes.toscrape.com/author/Charles-M-Schulz"}, "tag": ["chocolate", "food", "humor"]}, +{"text": "\u201cWe read to know we're not alone.\u201d", "author": {"name": "William Nicholson", "url": "http://quotes.toscrape.com/author/William-Nicholson"}, "tag": ["misattributed-to-c-s-lewis", "reading"]}, +{"text": "\u201cAny fool can know. The point is to understand.\u201d", "author": {"name": "Albert Einstein", "url": "http://quotes.toscrape.com/author/Albert-Einstein"}, "tag": ["knowledge", "learning", "understanding", "wisdom"]}, +{"text": "\u201cI have always imagined that Paradise will be a kind of library.\u201d", "author": {"name": "Jorge Luis Borges", "url": "http://quotes.toscrape.com/author/Jorge-Luis-Borges"}, "tag": ["books", "library"]}, +{"text": "\u201cIt is never too late to be what you might have been.\u201d", "author": {"name": "George Eliot", "url": "http://quotes.toscrape.com/author/George-Eliot"}, "tag": ["inspirational"]}, +{"text": "\u201cA reader lives a thousand lives before he dies, said Jojen. The man who never reads lives only one.\u201d", "author": {"name": "George R.R. Martin", "url": "http://quotes.toscrape.com/author/George-R-R-Martin"}, "tag": ["read", "readers", "reading", "reading-books"]}, +{"text": "\u201cYou can never get a cup of tea large enough or a book long enough to suit me.\u201d", "author": {"name": "C.S. Lewis", "url": "http://quotes.toscrape.com/author/C-S-Lewis"}, "tag": ["books", "inspirational", "reading", "tea"]}, +{"text": "\u201cYou believe lies so you eventually learn to trust no one but yourself.\u201d", "author": {"name": "Marilyn Monroe", "url": "http://quotes.toscrape.com/author/Marilyn-Monroe"}, "tag": []}, +{"text": "\u201cIf you can make a woman laugh, you can make her do anything.\u201d", "author": {"name": "Marilyn Monroe", "url": "http://quotes.toscrape.com/author/Marilyn-Monroe"}, "tag": ["girls", "love"]}, +{"text": "\u201cLife is like riding a bicycle. To keep your balance, you must keep moving.\u201d", "author": {"name": "Albert Einstein", "url": "http://quotes.toscrape.com/author/Albert-Einstein"}, "tag": ["life", "simile"]}, +{"text": "\u201cThe real lover is the man who can thrill you by kissing your forehead or smiling into your eyes or just staring into space.\u201d", "author": {"name": "Marilyn Monroe", "url": "http://quotes.toscrape.com/author/Marilyn-Monroe"}, "tag": ["love"]}, +{"text": "\u201cA wise girl kisses but doesn't love, listens but doesn't believe, and leaves before she is left.\u201d", "author": {"name": "Marilyn Monroe", "url": "http://quotes.toscrape.com/author/Marilyn-Monroe"}, "tag": ["attributed-no-source"]}, +{"text": "\u201cOnly in the darkness can you see the stars.\u201d", "author": {"name": "Martin Luther King Jr.", "url": "http://quotes.toscrape.com/author/Martin-Luther-King-Jr"}, "tag": ["hope", "inspirational"]}, +{"text": "\u201cIt matters not what someone is born, but what they grow to be.\u201d", "author": {"name": "J.K. Rowling", "url": "http://quotes.toscrape.com/author/J-K-Rowling"}, "tag": ["dumbledore"]}, +{"text": "\u201cLove does not begin and end the way we seem to think it does. Love is a battle, love is a war; love is a growing up.\u201d", "author": {"name": "James Baldwin", "url": "http://quotes.toscrape.com/author/James-Baldwin"}, "tag": ["love"]}, +{"text": "\u201cThere is nothing I would not do for those who are really my friends. I have no notion of loving people by halves, it is not my nature.\u201d", "author": {"name": "Jane Austen", "url": "http://quotes.toscrape.com/author/Jane-Austen"}, "tag": ["friendship", "love"]}, +{"text": "\u201cDo one thing every day that scares you.\u201d", "author": {"name": "Eleanor Roosevelt", "url": "http://quotes.toscrape.com/author/Eleanor-Roosevelt"}, "tag": ["attributed", "fear", "inspiration"]}, +{"text": "\u201cI am good, but not an angel. I do sin, but I am not the devil. I am just a small girl in a big world trying to find someone to love.\u201d", "author": {"name": "Marilyn Monroe", "url": "http://quotes.toscrape.com/author/Marilyn-Monroe"}, "tag": ["attributed-no-source"]}, +{"text": "\u201cIf I were not a physicist, I would probably be a musician. I often think in music. I live my daydreams in music. I see my life in terms of music.\u201d", "author": {"name": "Albert Einstein", "url": "http://quotes.toscrape.com/author/Albert-Einstein"}, "tag": ["music"]}, +{"text": "\u201cIf you only read the books that everyone else is reading, you can only think what everyone else is thinking.\u201d", "author": {"name": "Haruki Murakami", "url": "http://quotes.toscrape.com/author/Haruki-Murakami"}, "tag": ["books", "thought"]}, +{"text": "\u201cThe difference between genius and stupidity is: genius has its limits.\u201d", "author": {"name": "Alexandre Dumas fils", "url": "http://quotes.toscrape.com/author/Alexandre-Dumas-fils"}, "tag": ["misattributed-to-einstein"]}, +{"text": "\u201cHe's like a drug for you, Bella.\u201d", "author": {"name": "Stephenie Meyer", "url": "http://quotes.toscrape.com/author/Stephenie-Meyer"}, "tag": ["drug", "romance", "simile"]}, +{"text": "\u201cThere is no friend as loyal as a book.\u201d", "author": {"name": "Ernest Hemingway", "url": "http://quotes.toscrape.com/author/Ernest-Hemingway"}, "tag": ["books", "friends", "novelist-quotes"]}, +{"text": "\u201cWhen one door of happiness closes, another opens; but often we look so long at the closed door that we do not see the one which has been opened for us.\u201d", "author": {"name": "Helen Keller", "url": "http://quotes.toscrape.com/author/Helen-Keller"}, "tag": ["inspirational"]}, +{"text": "\u201cLife isn't about finding yourself. Life is about creating yourself.\u201d", "author": {"name": "George Bernard Shaw", "url": "http://quotes.toscrape.com/author/George-Bernard-Shaw"}, "tag": ["inspirational", "life", "yourself"]}, +{"text": "\u201cThat's the problem with drinking, I thought, as I poured myself a drink. If something bad happens you drink in an attempt to forget; if something good happens you drink in order to celebrate; and if nothing happens you drink to make something happen.\u201d", "author": {"name": "Charles Bukowski", "url": "http://quotes.toscrape.com/author/Charles-Bukowski"}, "tag": ["alcohol"]}, +{"text": "\u201cYou don\u2019t forget the face of the person who was your last hope.\u201d", "author": {"name": "Suzanne Collins", "url": "http://quotes.toscrape.com/author/Suzanne-Collins"}, "tag": ["the-hunger-games"]}, +{"text": "\u201cRemember, we're madly in love, so it's all right to kiss me anytime you feel like it.\u201d", "author": {"name": "Suzanne Collins", "url": "http://quotes.toscrape.com/author/Suzanne-Collins"}, "tag": ["humor"]}, +{"text": "\u201cTo love at all is to be vulnerable. Love anything and your heart will be wrung and possibly broken. If you want to make sure of keeping it intact you must give it to no one, not even an animal. Wrap it carefully round with hobbies and little luxuries; avoid all entanglements. Lock it up safe in the casket or coffin of your selfishness. But in that casket, safe, dark, motionless, airless, it will change. It will not be broken; it will become unbreakable, impenetrable, irredeemable. To love is to be vulnerable.\u201d", "author": {"name": "C.S. Lewis", "url": "http://quotes.toscrape.com/author/C-S-Lewis"}, "tag": ["love"]}, +{"text": "\u201cNot all those who wander are lost.\u201d", "author": {"name": "J.R.R. Tolkien", "url": "http://quotes.toscrape.com/author/J-R-R-Tolkien"}, "tag": ["bilbo", "journey", "lost", "quest", "travel", "wander"]}, +{"text": "\u201cDo not pity the dead, Harry. Pity the living, and, above all those who live without love.\u201d", "author": {"name": "J.K. Rowling", "url": "http://quotes.toscrape.com/author/J-K-Rowling"}, "tag": ["live-death-love"]}, +{"text": "\u201cThere is nothing to writing. All you do is sit down at a typewriter and bleed.\u201d", "author": {"name": "Ernest Hemingway", "url": "http://quotes.toscrape.com/author/Ernest-Hemingway"}, "tag": ["good", "writing"]}, +{"text": "\u201cFinish each day and be done with it. You have done what you could. Some blunders and absurdities no doubt crept in; forget them as soon as you can. Tomorrow is a new day. You shall begin it serenely and with too high a spirit to be encumbered with your old nonsense.\u201d", "author": {"name": "Ralph Waldo Emerson", "url": "http://quotes.toscrape.com/author/Ralph-Waldo-Emerson"}, "tag": ["life", "regrets"]}, +{"text": "\u201cI have never let my schooling interfere with my education.\u201d", "author": {"name": "Mark Twain", "url": "http://quotes.toscrape.com/author/Mark-Twain"}, "tag": ["education"]}, +{"text": "\u201cI have heard there are troubles of more than one kind. Some come from ahead and some come from behind. But I've bought a big bat. I'm all ready you see. Now my troubles are going to have troubles with me!\u201d", "author": {"name": "Dr. Seuss", "url": "http://quotes.toscrape.com/author/Dr-Seuss"}, "tag": ["troubles"]}, +{"text": "\u201cIf I had a flower for every time I thought of you...I could walk through my garden forever.\u201d", "author": {"name": "Alfred Tennyson", "url": "http://quotes.toscrape.com/author/Alfred-Tennyson"}, "tag": ["friendship", "love"]}, +{"text": "\u201cSome people never go crazy. What truly horrible lives they must lead.\u201d", "author": {"name": "Charles Bukowski", "url": "http://quotes.toscrape.com/author/Charles-Bukowski"}, "tag": ["humor"]}, +{"text": "\u201cThe trouble with having an open mind, of course, is that people will insist on coming along and trying to put things in it.\u201d", "author": {"name": "Terry Pratchett", "url": "http://quotes.toscrape.com/author/Terry-Pratchett"}, "tag": ["humor", "open-mind", "thinking"]}, +{"text": "\u201cThink left and think right and think low and think high. Oh, the thinks you can think up if only you try!\u201d", "author": {"name": "Dr. Seuss", "url": "http://quotes.toscrape.com/author/Dr-Seuss"}, "tag": ["humor", "philosophy"]}, +{"text": "\u201cWhat really knocks me out is a book that, when you're all done reading it, you wish the author that wrote it was a terrific friend of yours and you could call him up on the phone whenever you felt like it. That doesn't happen much, though.\u201d", "author": {"name": "J.D. Salinger", "url": "http://quotes.toscrape.com/author/J-D-Salinger"}, "tag": ["authors", "books", "literature", "reading", "writing"]}, +{"text": "\u201cThe reason I talk to myself is because I\u2019m the only one whose answers I accept.\u201d", "author": {"name": "George Carlin", "url": "http://quotes.toscrape.com/author/George-Carlin"}, "tag": ["humor", "insanity", "lies", "lying", "self-indulgence", "truth"]}, +{"text": "\u201cYou may say I'm a dreamer, but I'm not the only one. I hope someday you'll join us. And the world will live as one.\u201d", "author": {"name": "John Lennon", "url": "http://quotes.toscrape.com/author/John-Lennon"}, "tag": ["beatles", "connection", "dreamers", "dreaming", "dreams", "hope", "inspirational", "peace"]}, +{"text": "\u201cI am free of all prejudice. I hate everyone equally. \u201d", "author": {"name": "W.C. Fields", "url": "http://quotes.toscrape.com/author/W-C-Fields"}, "tag": ["humor", "sinister"]}, +{"text": "\u201cThe question isn't who is going to let me; it's who is going to stop me.\u201d", "author": {"name": "Ayn Rand", "url": "http://quotes.toscrape.com/author/Ayn-Rand"}, "tag": []}, +{"text": "\u201c\u2032Classic\u2032 - a book which people praise and don't read.\u201d", "author": {"name": "Mark Twain", "url": "http://quotes.toscrape.com/author/Mark-Twain"}, "tag": ["books", "classic", "reading"]}, +{"text": "\u201cAnyone who has never made a mistake has never tried anything new.\u201d", "author": {"name": "Albert Einstein", "url": "http://quotes.toscrape.com/author/Albert-Einstein"}, "tag": ["mistakes"]}, +{"text": "\u201cA lady's imagination is very rapid; it jumps from admiration to love, from love to matrimony in a moment.\u201d", "author": {"name": "Jane Austen", "url": "http://quotes.toscrape.com/author/Jane-Austen"}, "tag": ["humor", "love", "romantic", "women"]}, +{"text": "\u201cRemember, if the time should come when you have to make a choice between what is right and what is easy, remember what happened to a boy who was good, and kind, and brave, because he strayed across the path of Lord Voldemort. Remember Cedric Diggory.\u201d", "author": {"name": "J.K. Rowling", "url": "http://quotes.toscrape.com/author/J-K-Rowling"}, "tag": ["integrity"]}, +{"text": "\u201cI declare after all there is no enjoyment like reading! How much sooner one tires of any thing than of a book! -- When I have a house of my own, I shall be miserable if I have not an excellent library.\u201d", "author": {"name": "Jane Austen", "url": "http://quotes.toscrape.com/author/Jane-Austen"}, "tag": ["books", "library", "reading"]}, +{"text": "\u201cThere are few people whom I really love, and still fewer of whom I think well. The more I see of the world, the more am I dissatisfied with it; and every day confirms my belief of the inconsistency of all human characters, and of the little dependence that can be placed on the appearance of merit or sense.\u201d", "author": {"name": "Jane Austen", "url": "http://quotes.toscrape.com/author/Jane-Austen"}, "tag": ["elizabeth-bennet", "jane-austen"]}, +{"text": "\u201cSome day you will be old enough to start reading fairy tales again.\u201d", "author": {"name": "C.S. Lewis", "url": "http://quotes.toscrape.com/author/C-S-Lewis"}, "tag": ["age", "fairytales", "growing-up"]}, +{"text": "\u201cWe are not necessarily doubting that God will do the best for us; we are wondering how painful the best will turn out to be.\u201d", "author": {"name": "C.S. Lewis", "url": "http://quotes.toscrape.com/author/C-S-Lewis"}, "tag": ["god"]}, +{"text": "\u201cThe fear of death follows from the fear of life. A man who lives fully is prepared to die at any time.\u201d", "author": {"name": "Mark Twain", "url": "http://quotes.toscrape.com/author/Mark-Twain"}, "tag": ["death", "life"]}, +{"text": "\u201cA lie can travel half way around the world while the truth is putting on its shoes.\u201d", "author": {"name": "Mark Twain", "url": "http://quotes.toscrape.com/author/Mark-Twain"}, "tag": ["misattributed-mark-twain", "truth"]}, +{"text": "\u201cI believe in Christianity as I believe that the sun has risen: not only because I see it, but because by it I see everything else.\u201d", "author": {"name": "C.S. Lewis", "url": "http://quotes.toscrape.com/author/C-S-Lewis"}, "tag": ["christianity", "faith", "religion", "sun"]}, +{"text": "\u201cThe truth.\" Dumbledore sighed. \"It is a beautiful and terrible thing, and should therefore be treated with great caution.\u201d", "author": {"name": "J.K. Rowling", "url": "http://quotes.toscrape.com/author/J-K-Rowling"}, "tag": ["truth"]}, +{"text": "\u201cI'm the one that's got to die when it's time for me to die, so let me live my life the way I want to.\u201d", "author": {"name": "Jimi Hendrix", "url": "http://quotes.toscrape.com/author/Jimi-Hendrix"}, "tag": ["death", "life"]}, +{"text": "\u201cTo die will be an awfully big adventure.\u201d", "author": {"name": "J.M. Barrie", "url": "http://quotes.toscrape.com/author/J-M-Barrie"}, "tag": ["adventure", "love"]}, +{"text": "\u201cIt takes courage to grow up and become who you really are.\u201d", "author": {"name": "E.E. Cummings", "url": "http://quotes.toscrape.com/author/E-E-Cummings"}, "tag": ["courage"]}, +{"text": "\u201cBut better to get hurt by the truth than comforted with a lie.\u201d", "author": {"name": "Khaled Hosseini", "url": "http://quotes.toscrape.com/author/Khaled-Hosseini"}, "tag": ["life"]}, +{"text": "\u201cYou never really understand a person until you consider things from his point of view... Until you climb inside of his skin and walk around in it.\u201d", "author": {"name": "Harper Lee", "url": "http://quotes.toscrape.com/author/Harper-Lee"}, "tag": ["better-life-empathy"]}, +{"text": "\u201cYou have to write the book that wants to be written. And if the book will be too difficult for grown-ups, then you write it for children.\u201d", "author": {"name": "Madeleine L'Engle", "url": "http://quotes.toscrape.com/author/Madeleine-LEngle"}, "tag": ["books", "children", "difficult", "grown-ups", "write", "writers", "writing"]}, +{"text": "\u201cNever tell the truth to people who are not worthy of it.\u201d", "author": {"name": "Mark Twain", "url": "http://quotes.toscrape.com/author/Mark-Twain"}, "tag": ["truth"]}, +{"text": "\u201cA person's a person, no matter how small.\u201d", "author": {"name": "Dr. Seuss", "url": "http://quotes.toscrape.com/author/Dr-Seuss"}, "tag": ["inspirational"]}, +{"text": "\u201c... a mind needs books as a sword needs a whetstone, if it is to keep its edge.\u201d", "author": {"name": "George R.R. Martin", "url": "http://quotes.toscrape.com/author/George-R-R-Martin"}, "tag": ["books", "mind"]} +] \ No newline at end of file diff --git a/WebCrawler/postscrape/postscrape/__init__.py b/WebCrawler/postscrape/postscrape/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/WebCrawler/postscrape/postscrape/__pycache__/__init__.cpython-38.pyc b/WebCrawler/postscrape/postscrape/__pycache__/__init__.cpython-38.pyc new file mode 100644 index 0000000..49c675b Binary files /dev/null and b/WebCrawler/postscrape/postscrape/__pycache__/__init__.cpython-38.pyc differ diff --git a/WebCrawler/postscrape/postscrape/__pycache__/items.cpython-38.pyc b/WebCrawler/postscrape/postscrape/__pycache__/items.cpython-38.pyc new file mode 100644 index 0000000..703bb60 Binary files /dev/null and b/WebCrawler/postscrape/postscrape/__pycache__/items.cpython-38.pyc differ diff --git a/WebCrawler/postscrape/postscrape/__pycache__/pipelines.cpython-38.pyc b/WebCrawler/postscrape/postscrape/__pycache__/pipelines.cpython-38.pyc new file mode 100644 index 0000000..353e8d9 Binary files /dev/null and b/WebCrawler/postscrape/postscrape/__pycache__/pipelines.cpython-38.pyc differ diff --git a/WebCrawler/postscrape/postscrape/__pycache__/settings.cpython-38.pyc b/WebCrawler/postscrape/postscrape/__pycache__/settings.cpython-38.pyc new file mode 100644 index 0000000..126753b Binary files /dev/null and b/WebCrawler/postscrape/postscrape/__pycache__/settings.cpython-38.pyc differ diff --git a/WebCrawler/postscrape/postscrape/items.py b/WebCrawler/postscrape/postscrape/items.py new file mode 100644 index 0000000..822144b --- /dev/null +++ b/WebCrawler/postscrape/postscrape/items.py @@ -0,0 +1,14 @@ +# Define here the models for your scraped items +# +# See documentation in: +# https://docs.scrapy.org/en/latest/topics/items.html + +import scrapy + + +class PostscrapeItem(scrapy.Item): + + # define the fields for your item here like: + text = scrapy.Field() + author = scrapy.Field() + tag = scrapy.Field() diff --git a/WebCrawler/postscrape/postscrape/middlewares.py b/WebCrawler/postscrape/postscrape/middlewares.py new file mode 100644 index 0000000..c847ca6 --- /dev/null +++ b/WebCrawler/postscrape/postscrape/middlewares.py @@ -0,0 +1,103 @@ +# Define here the models for your spider middleware +# +# See documentation in: +# https://docs.scrapy.org/en/latest/topics/spider-middleware.html + +from scrapy import signals + +# useful for handling different item types with a single interface +from itemadapter import is_item, ItemAdapter + + +class PostscrapeSpiderMiddleware: + # Not all methods need to be defined. If a method is not defined, + # scrapy acts as if the spider middleware does not modify the + # passed objects. + + @classmethod + def from_crawler(cls, crawler): + # This method is used by Scrapy to create your spiders. + s = cls() + crawler.signals.connect(s.spider_opened, signal=signals.spider_opened) + return s + + def process_spider_input(self, response, spider): + # Called for each response that goes through the spider + # middleware and into the spider. + + # Should return None or raise an exception. + return None + + def process_spider_output(self, response, result, spider): + # Called with the results returned from the Spider, after + # it has processed the response. + + # Must return an iterable of Request, or item objects. + for i in result: + yield i + + def process_spider_exception(self, response, exception, spider): + # Called when a spider or process_spider_input() method + # (from other spider middleware) raises an exception. + + # Should return either None or an iterable of Request or item objects. + pass + + def process_start_requests(self, start_requests, spider): + # Called with the start requests of the spider, and works + # similarly to the process_spider_output() method, except + # that it doesn’t have a response associated. + + # Must return only requests (not items). + for r in start_requests: + yield r + + def spider_opened(self, spider): + spider.logger.info('Spider opened: %s' % spider.name) + + +class PostscrapeDownloaderMiddleware: + # Not all methods need to be defined. If a method is not defined, + # scrapy acts as if the downloader middleware does not modify the + # passed objects. + + @classmethod + def from_crawler(cls, crawler): + # This method is used by Scrapy to create your spiders. + s = cls() + crawler.signals.connect(s.spider_opened, signal=signals.spider_opened) + return s + + def process_request(self, request, spider): + # Called for each request that goes through the downloader + # middleware. + + # Must either: + # - return None: continue processing this request + # - or return a Response object + # - or return a Request object + # - or raise IgnoreRequest: process_exception() methods of + # installed downloader middleware will be called + return None + + def process_response(self, request, response, spider): + # Called with the response returned from the downloader. + + # Must either; + # - return a Response object + # - return a Request object + # - or raise IgnoreRequest + return response + + def process_exception(self, request, exception, spider): + # Called when a download handler or a process_request() + # (from other downloader middleware) raises an exception. + + # Must either: + # - return None: continue processing this exception + # - return a Response object: stops process_exception() chain + # - return a Request object: stops process_exception() chain + pass + + def spider_opened(self, spider): + spider.logger.info('Spider opened: %s' % spider.name) diff --git a/WebCrawler/postscrape/postscrape/pipelines.py b/WebCrawler/postscrape/postscrape/pipelines.py new file mode 100644 index 0000000..db446d6 --- /dev/null +++ b/WebCrawler/postscrape/postscrape/pipelines.py @@ -0,0 +1,20 @@ +# Define your item pipelines here +# +# Don't forget to add your pipeline to the ITEM_PIPELINES setting +# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html +import pymongo + +class PostscrapePipeline(object): + + def __init__(self): + self.conn =pymongo.MongoClient( + 'localhost', + 27017 + ) + db = self.conn['quotestoscrape'] + self.collection = db['alexander_parreira'] + + def process_item(self, item, spider): + + self.collection.insert(dict(item)) + return item diff --git a/WebCrawler/postscrape/postscrape/settings.py b/WebCrawler/postscrape/postscrape/settings.py new file mode 100644 index 0000000..d15b604 --- /dev/null +++ b/WebCrawler/postscrape/postscrape/settings.py @@ -0,0 +1,88 @@ +# Scrapy settings for postscrape project +# +# For simplicity, this file contains only settings considered important or +# commonly used. You can find more settings consulting the documentation: +# +# https://docs.scrapy.org/en/latest/topics/settings.html +# https://docs.scrapy.org/en/latest/topics/downloader-middleware.html +# https://docs.scrapy.org/en/latest/topics/spider-middleware.html + +BOT_NAME = 'postscrape' + +SPIDER_MODULES = ['postscrape.spiders'] +NEWSPIDER_MODULE = 'postscrape.spiders' + + +# Crawl responsibly by identifying yourself (and your website) on the user-agent +#USER_AGENT = 'postscrape (+http://www.yourdomain.com)' + +# Obey robots.txt rules +ROBOTSTXT_OBEY = True + +# Configure maximum concurrent requests performed by Scrapy (default: 16) +#CONCURRENT_REQUESTS = 32 + +# Configure a delay for requests for the same website (default: 0) +# See https://docs.scrapy.org/en/latest/topics/settings.html#download-delay +# See also autothrottle settings and docs +#DOWNLOAD_DELAY = 3 +# The download delay setting will honor only one of: +#CONCURRENT_REQUESTS_PER_DOMAIN = 16 +#CONCURRENT_REQUESTS_PER_IP = 16 + +# Disable cookies (enabled by default) +#COOKIES_ENABLED = False + +# Disable Telnet Console (enabled by default) +#TELNETCONSOLE_ENABLED = False + +# Override the default request headers: +#DEFAULT_REQUEST_HEADERS = { +# 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', +# 'Accept-Language': 'en', +#} + +# Enable or disable spider middlewares +# See https://docs.scrapy.org/en/latest/topics/spider-middleware.html +#SPIDER_MIDDLEWARES = { +# 'postscrape.middlewares.PostscrapeSpiderMiddleware': 543, +#} + +# Enable or disable downloader middlewares +# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html +#DOWNLOADER_MIDDLEWARES = { +# 'postscrape.middlewares.PostscrapeDownloaderMiddleware': 543, +#} + +# Enable or disable extensions +# See https://docs.scrapy.org/en/latest/topics/extensions.html +#EXTENSIONS = { +# 'scrapy.extensions.telnet.TelnetConsole': None, +#} + +# Configure item pipelines +# See https://docs.scrapy.org/en/latest/topics/item-pipeline.html +ITEM_PIPELINES = { + 'postscrape.pipelines.PostscrapePipeline': 300, +} + +# Enable and configure the AutoThrottle extension (disabled by default) +# See https://docs.scrapy.org/en/latest/topics/autothrottle.html +#AUTOTHROTTLE_ENABLED = True +# The initial download delay +#AUTOTHROTTLE_START_DELAY = 5 +# The maximum download delay to be set in case of high latencies +#AUTOTHROTTLE_MAX_DELAY = 60 +# The average number of requests Scrapy should be sending in parallel to +# each remote server +#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0 +# Enable showing throttling stats for every response received: +#AUTOTHROTTLE_DEBUG = False + +# Enable and configure HTTP caching (disabled by default) +# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings +#HTTPCACHE_ENABLED = True +#HTTPCACHE_EXPIRATION_SECS = 0 +#HTTPCACHE_DIR = 'httpcache' +#HTTPCACHE_IGNORE_HTTP_CODES = [] +#HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage' diff --git a/WebCrawler/postscrape/postscrape/spiders/__init__.py b/WebCrawler/postscrape/postscrape/spiders/__init__.py new file mode 100644 index 0000000..ebd689a --- /dev/null +++ b/WebCrawler/postscrape/postscrape/spiders/__init__.py @@ -0,0 +1,4 @@ +# This package will contain the spiders of your Scrapy project +# +# Please refer to the documentation for information on how to create and manage +# your spiders. diff --git a/WebCrawler/postscrape/postscrape/spiders/__pycache__/__init__.cpython-38.pyc b/WebCrawler/postscrape/postscrape/spiders/__pycache__/__init__.cpython-38.pyc new file mode 100644 index 0000000..25d01a6 Binary files /dev/null and b/WebCrawler/postscrape/postscrape/spiders/__pycache__/__init__.cpython-38.pyc differ diff --git a/WebCrawler/postscrape/postscrape/spiders/__pycache__/posts-spider.cpython-38.pyc b/WebCrawler/postscrape/postscrape/spiders/__pycache__/posts-spider.cpython-38.pyc new file mode 100644 index 0000000..58d757c Binary files /dev/null and b/WebCrawler/postscrape/postscrape/spiders/__pycache__/posts-spider.cpython-38.pyc differ diff --git a/WebCrawler/postscrape/postscrape/spiders/__pycache__/queries.cpython-38.pyc b/WebCrawler/postscrape/postscrape/spiders/__pycache__/queries.cpython-38.pyc new file mode 100644 index 0000000..f0cd181 Binary files /dev/null and b/WebCrawler/postscrape/postscrape/spiders/__pycache__/queries.cpython-38.pyc differ diff --git a/WebCrawler/postscrape/postscrape/spiders/posts-spider.py b/WebCrawler/postscrape/postscrape/spiders/posts-spider.py new file mode 100644 index 0000000..f972b14 --- /dev/null +++ b/WebCrawler/postscrape/postscrape/spiders/posts-spider.py @@ -0,0 +1,44 @@ +import scrapy +from ..items import PostscrapeItem + +class TitleSpider(scrapy.Spider): + name = "titles" + start_urls = [ + 'http://quotes.toscrape.com/', + ] + + def parse(self, response): + + itens = PostscrapeItem() + + div_titles = response.css('div.quote') + + for title in div_titles: + text = title.css('span.text::text').get() + author = title.css('span small::text').get() + link = title.css('span a::attr(href)').get() + tag = title.css('div.tags a.tag::text').getall() + + itens['text'] = text + itens['author'] = {'name': author , + 'url':'http://quotes.toscrape.com'+link} + itens['tag'] = tag + + yield itens + + next_page = response.css('li.next a::attr(href)').get() + if next_page is not None: + yield response.follow(next_page, callback=self.parse) + + + + """ + page = response.url.split('/')[-1] + filename = 'posts-%s.html' % page + with open(filename, 'wb') as f: + f.write(response.body) + + + class titlesSpider(scrapy.Spider): + + """ \ No newline at end of file diff --git a/WebCrawler/postscrape/postscrape/spiders/queries.js b/WebCrawler/postscrape/postscrape/spiders/queries.js new file mode 100644 index 0000000..8ef04e4 --- /dev/null +++ b/WebCrawler/postscrape/postscrape/spiders/queries.js @@ -0,0 +1,27 @@ + + //Quantas citações por autor foram coletadas? + db.alexander_parreira.aggregate([ + {$group: + {_id: '$author.name', + qtd:{$sum: 1} + } + }, + {$sort: {qtd: -1} + + }]).pretty() + + + //Quantas tags distintas foram coletadas? + db.alexander_parreira.aggregate([ + {$unwind: { + path:'$tag', + }}, {$group: { + _id: '$tag', + }}, {$count: 'tag'} + ]).pretty() + + //Quantas citações foram coletadas? + db.alexander_parreira.aggregate([ + {$group: { + _id: '$text',} + }, {$count: 'tag'}]) \ No newline at end of file diff --git a/WebCrawler/postscrape/scrapy.cfg b/WebCrawler/postscrape/scrapy.cfg new file mode 100644 index 0000000..0c58b8c --- /dev/null +++ b/WebCrawler/postscrape/scrapy.cfg @@ -0,0 +1,11 @@ +# Automatically created by: scrapy startproject +# +# For more information about the [deploy] section see: +# https://scrapyd.readthedocs.io/en/latest/deploy.html + +[settings] +default = postscrape.settings + +[deploy] +#url = http://localhost:6800/ +project = postscrape diff --git a/WebCrawler/venv/Lib/site-packages/Automat-20.2.0.dist-info/INSTALLER b/WebCrawler/venv/Lib/site-packages/Automat-20.2.0.dist-info/INSTALLER new file mode 100644 index 0000000..a1b589e --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/Automat-20.2.0.dist-info/INSTALLER @@ -0,0 +1 @@ +pip diff --git a/WebCrawler/venv/Lib/site-packages/Automat-20.2.0.dist-info/LICENSE b/WebCrawler/venv/Lib/site-packages/Automat-20.2.0.dist-info/LICENSE new file mode 100644 index 0000000..9773501 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/Automat-20.2.0.dist-info/LICENSE @@ -0,0 +1,21 @@ +Copyright (c) 2014 +Rackspace + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/WebCrawler/venv/Lib/site-packages/Automat-20.2.0.dist-info/METADATA b/WebCrawler/venv/Lib/site-packages/Automat-20.2.0.dist-info/METADATA new file mode 100644 index 0000000..8b4c9ce --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/Automat-20.2.0.dist-info/METADATA @@ -0,0 +1,487 @@ +Metadata-Version: 2.1 +Name: Automat +Version: 20.2.0 +Summary: Self-service finite-state machines for the programmer on the go. +Home-page: https://github.com/glyph/Automat +Author: Glyph +Author-email: glyph@twistedmatrix.com +License: MIT +Keywords: fsm finite state machine automata +Platform: UNKNOWN +Classifier: Intended Audience :: Developers +Classifier: License :: OSI Approved :: MIT License +Classifier: Operating System :: OS Independent +Classifier: Programming Language :: Python +Classifier: Programming Language :: Python :: 2 +Classifier: Programming Language :: Python :: 2.7 +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3.5 +Classifier: Programming Language :: Python :: 3.6 +Classifier: Programming Language :: Python :: 3.7 +Classifier: Programming Language :: Python :: 3.8 +Requires-Dist: attrs (>=19.2.0) +Requires-Dist: six +Provides-Extra: visualize +Requires-Dist: graphviz (>0.5.1); extra == 'visualize' +Requires-Dist: Twisted (>=16.1.1); extra == 'visualize' + + +Automat +======= + + +.. image:: https://readthedocs.org/projects/automat/badge/?version=latest + :target: http://automat.readthedocs.io/en/latest/ + :alt: Documentation Status + + +.. image:: https://travis-ci.org/glyph/automat.svg?branch=master + :target: https://travis-ci.org/glyph/automat + :alt: Build Status + + +.. image:: https://coveralls.io/repos/glyph/automat/badge.png + :target: https://coveralls.io/r/glyph/automat + :alt: Coverage Status + + +Self-service finite-state machines for the programmer on the go. +---------------------------------------------------------------- + +Automat is a library for concise, idiomatic Python expression of finite-state +automata (particularly deterministic finite-state transducers). + +Read more here, or on `Read the Docs `_\ , or watch the following videos for an overview and presentation + +Overview and presentation by **Glyph Lefkowitz** at the first talk of the first Pyninsula meetup, on February 21st, 2017: + +.. image:: https://img.youtube.com/vi/0wOZBpD1VVk/0.jpg + :target: https://www.youtube.com/watch?v=0wOZBpD1VVk + :alt: Glyph Lefkowitz - Automat - Pyninsula #0 + + +Presentation by **Clinton Roy** at PyCon Australia, on August 6th 2017: + +.. image:: https://img.youtube.com/vi/TedUKXhu9kE/0.jpg + :target: https://www.youtube.com/watch?v=TedUKXhu9kE + :alt: Clinton Roy - State Machines - Pycon Australia 2017 + + +Why use state machines? +^^^^^^^^^^^^^^^^^^^^^^^ + +Sometimes you have to create an object whose behavior varies with its state, +but still wishes to present a consistent interface to its callers. + +For example, let's say you're writing the software for a coffee machine. It +has a lid that can be opened or closed, a chamber for water, a chamber for +coffee beans, and a button for "brew". + +There are a number of possible states for the coffee machine. It might or +might not have water. It might or might not have beans. The lid might be open +or closed. The "brew" button should only actually attempt to brew coffee in +one of these configurations, and the "open lid" button should only work if the +coffee is not, in fact, brewing. + +With diligence and attention to detail, you can implement this correctly using +a collection of attributes on an object; ``has_water``\ , ``has_beans``\ , +``is_lid_open`` and so on. However, you have to keep all these attributes +consistent. As the coffee maker becomes more complex - perhaps you add an +additional chamber for flavorings so you can make hazelnut coffee, for +example - you have to keep adding more and more checks and more and more +reasoning about which combinations of states are allowed. + +Rather than adding tedious 'if' checks to every single method to make sure that +each of these flags are exactly what you expect, you can use a state machine to +ensure that if your code runs at all, it will be run with all the required +values initialized, because they have to be called in the order you declare +them. + +You can read about state machines and their advantages for Python programmers +in considerably more detail +`in this excellent series of articles from ClusterHQ `_. + +What makes Automat different? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +There are +`dozens of libraries on PyPI implementing state machines `_. +So it behooves me to say why yet another one would be a good idea. + +Automat is designed around this principle: while organizing your code around +state machines is a good idea, your callers don't, and shouldn't have to, care +that you've done so. In Python, the "input" to a stateful system is a method +call; the "output" may be a method call, if you need to invoke a side effect, +or a return value, if you are just performing a computation in memory. Most +other state-machine libraries require you to explicitly create an input object, +provide that object to a generic "input" method, and then receive results, +sometimes in terms of that library's interfaces and sometimes in terms of +classes you define yourself. + +For example, a snippet of the coffee-machine example above might be implemented +as follows in naive Python: + +.. code-block:: python + + class CoffeeMachine(object): + def brew_button(self): + if self.has_water and self.has_beans and not self.is_lid_open: + self.heat_the_heating_element() + # ... + +With Automat, you'd create a class with a ``MethodicalMachine`` attribute: + +.. code-block:: python + + from automat import MethodicalMachine + + class CoffeeBrewer(object): + _machine = MethodicalMachine() + +and then you would break the above logic into two pieces - the ``brew_button`` +*input*\ , declared like so: + +.. code-block:: python + + @_machine.input() + def brew_button(self): + "The user pressed the 'brew' button." + +It wouldn't do any good to declare a method *body* on this, however, because +input methods don't actually execute their bodies when called; doing actual +work is the *output*\ 's job: + +.. code-block:: python + + @_machine.output() + def _heat_the_heating_element(self): + "Heat up the heating element, which should cause coffee to happen." + self._heating_element.turn_on() + +As well as a couple of *states* - and for simplicity's sake let's say that the +only two states are ``have_beans`` and ``dont_have_beans``\ : + +.. code-block:: python + + @_machine.state() + def have_beans(self): + "In this state, you have some beans." + @_machine.state(initial=True) + def dont_have_beans(self): + "In this state, you don't have any beans." + +``dont_have_beans`` is the ``initial`` state because ``CoffeeBrewer`` starts without beans +in it. + +(And another input to put some beans in:) + +.. code-block:: python + + @_machine.input() + def put_in_beans(self): + "The user put in some beans." + +Finally, you hook everything together with the ``upon`` method of the functions +decorated with ``_machine.state``\ : + +.. code-block:: python + + + # When we don't have beans, upon putting in beans, we will then have beans + # (and produce no output) + dont_have_beans.upon(put_in_beans, enter=have_beans, outputs=[]) + + # When we have beans, upon pressing the brew button, we will then not have + # beans any more (as they have been entered into the brewing chamber) and + # our output will be heating the heating element. + have_beans.upon(brew_button, enter=dont_have_beans, + outputs=[_heat_the_heating_element]) + +To *users* of this coffee machine class though, it still looks like a POPO +(Plain Old Python Object): + +.. code-block:: python + + >>> coffee_machine = CoffeeMachine() + >>> coffee_machine.put_in_beans() + >>> coffee_machine.brew_button() + +All of the *inputs* are provided by calling them like methods, all of the +*outputs* are automatically invoked when they are produced according to the +outputs specified to ``upon`` and all of the states are simply opaque tokens - +although the fact that they're defined as methods like inputs and outputs +allows you to put docstrings on them easily to document them. + +How do I get the current state of a state machine? +-------------------------------------------------- + +Don't do that. + +One major reason for having a state machine is that you want the callers of the +state machine to just provide the appropriate input to the machine at the +appropriate time, and *not have to check themselves* what state the machine is +in. So if you are tempted to write some code like this: + +.. code-block:: python + + if connection_state_machine.state == "CONNECTED": + connection_state_machine.send_message() + else: + print("not connected") + +Instead, just make your calling code do this: + +.. code-block:: python + + connection_state_machine.send_message() + +and then change your state machine to look like this: + +.. code-block:: python + + @_machine.state() + def connected(self): + "connected" + @_machine.state() + def not_connected(self): + "not connected" + @_machine.input() + def send_message(self): + "send a message" + @_machine.output() + def _actually_send_message(self): + self._transport.send(b"message") + @_machine.output() + def _report_sending_failure(self): + print("not connected") + connected.upon(send_message, enter=connected, [_actually_send_message]) + not_connected.upon(send_message, enter=not_connected, [_report_sending_failure]) + +so that the responsibility for knowing which state the state machine is in +remains within the state machine itself. + +Input for Inputs and Output for Outputs +--------------------------------------- + +Quite often you want to be able to pass parameters to your methods, as well as +inspecting their results. For example, when you brew the coffee, you might +expect a cup of coffee to result, and you would like to see what kind of coffee +it is. And if you were to put delicious hand-roasted small-batch artisanal +beans into the machine, you would expect a *better* cup of coffee than if you +were to use mass-produced beans. You would do this in plain old Python by +adding a parameter, so that's how you do it in Automat as well. + +.. code-block:: python + + @_machine.input() + def put_in_beans(self, beans): + "The user put in some beans." + +However, one important difference here is that *we can't add any +implementation code to the input method*. Inputs are purely a declaration of +the interface; the behavior must all come from outputs. Therefore, the change +in the state of the coffee machine must be represented as an output. We can +add an output method like this: + +.. code-block:: python + + @_machine.output() + def _save_beans(self, beans): + "The beans are now in the machine; save them." + self._beans = beans + +and then connect it to the ``put_in_beans`` by changing the transition from +``dont_have_beans`` to ``have_beans`` like so: + +.. code-block:: python + + dont_have_beans.upon(put_in_beans, enter=have_beans, + outputs=[_save_beans]) + +Now, when you call: + +.. code-block:: python + + coffee_machine.put_in_beans("real good beans") + +the machine will remember the beans for later. + +So how do we get the beans back out again? One of our outputs needs to have a +return value. It would make sense if our ``brew_button`` method returned the cup +of coffee that it made, so we should add an output. So, in addition to heating +the heating element, let's add a return value that describes the coffee. First +a new output: + +.. code-block:: python + + @_machine.output() + def _describe_coffee(self): + return "A cup of coffee made with {}.".format(self._beans) + +Note that we don't need to check first whether ``self._beans`` exists or not, +because we can only reach this output method if the state machine says we've +gone through a set of states that sets this attribute. + +Now, we need to hook up ``_describe_coffee`` to the process of brewing, so change +the brewing transition to: + +.. code-block:: python + + have_beans.upon(brew_button, enter=dont_have_beans, + outputs=[_heat_the_heating_element, + _describe_coffee]) + +Now, we can call it: + +.. code-block:: python + + >>> coffee_machine.brew_button() + [None, 'A cup of coffee made with real good beans.'] + +Except... wait a second, what's that ``None`` doing there? + +Since every input can produce multiple outputs, in automat, the default return +value from every input invocation is a ``list``. In this case, we have both +``_heat_the_heating_element`` and ``_describe_coffee`` outputs, so we're seeing +both of their return values. However, this can be customized, with the +``collector`` argument to ``upon``\ ; the ``collector`` is a callable which takes an +iterable of all the outputs' return values and "collects" a single return value +to return to the caller of the state machine. + +In this case, we only care about the last output, so we can adjust the call to +``upon`` like this: + +.. code-block:: python + + have_beans.upon(brew_button, enter=dont_have_beans, + outputs=[_heat_the_heating_element, + _describe_coffee], + collector=lambda iterable: list(iterable)[-1] + ) + +And now, we'll get just the return value we want: + +.. code-block:: python + + >>> coffee_machine.brew_button() + 'A cup of coffee made with real good beans.' + +If I can't get the state of the state machine, how can I save it to (a database, an API response, a file on disk...) +-------------------------------------------------------------------------------------------------------------------- + +There are APIs for serializing the state machine. + +First, you have to decide on a persistent representation of each state, via the +``serialized=`` argument to the ``MethodicalMachine.state()`` decorator. + +Let's take this very simple "light switch" state machine, which can be on or +off, and flipped to reverse its state: + +.. code-block:: python + + class LightSwitch(object): + _machine = MethodicalMachine() + @_machine.state(serialized="on") + def on_state(self): + "the switch is on" + @_machine.state(serialized="off", initial=True) + def off_state(self): + "the switch is off" + @_machine.input() + def flip(self): + "flip the switch" + on_state.upon(flip, enter=off_state, outputs=[]) + off_state.upon(flip, enter=on_state, outputs=[]) + +In this case, we've chosen a serialized representation for each state via the +``serialized`` argument. The on state is represented by the string ``"on"``\ , and +the off state is represented by the string ``"off"``. + +Now, let's just add an input that lets us tell if the switch is on or not. + +.. code-block:: python + + @_machine.input() + def query_power(self): + "return True if powered, False otherwise" + @_machine.output() + def _is_powered(self): + return True + @_machine.output() + def _not_powered(self): + return False + on_state.upon(query_power, enter=on_state, outputs=[_is_powered], + collector=next) + off_state.upon(query_power, enter=off_state, outputs=[_not_powered], + collector=next) + +To save the state, we have the ``MethodicalMachine.serializer()`` method. A +method decorated with ``@serializer()`` gets an extra argument injected at the +beginning of its argument list: the serialized identifier for the state. In +this case, either ``"on"`` or ``"off"``. Since state machine output methods can +also affect other state on the object, a serializer method is expected to +return *all* relevant state for serialization. + +For our simple light switch, such a method might look like this: + +.. code-block:: python + + @_machine.serializer() + def save(self, state): + return {"is-it-on": state} + +Serializers can be public methods, and they can return whatever you like. If +necessary, you can have different serializers - just multiple methods decorated +with ``@_machine.serializer()`` - for different formats; return one data-structure +for JSON, one for XML, one for a database row, and so on. + +When it comes time to unserialize, though, you generally want a private method, +because an unserializer has to take a not-fully-initialized instance and +populate it with state. It is expected to *return* the serialized machine +state token that was passed to the serializer, but it can take whatever +arguments you like. Of course, in order to return that, it probably has to +take it somewhere in its arguments, so it will generally take whatever a paired +serializer has returned as an argument. + +So our unserializer would look like this: + +.. code-block:: python + + @_machine.unserializer() + def _restore(self, blob): + return blob["is-it-on"] + +Generally you will want a classmethod deserialization constructor which you +write yourself to call this, so that you know how to create an instance of your +own object, like so: + +.. code-block:: python + + @classmethod + def from_blob(cls, blob): + self = cls() + self._restore(blob) + return self + +Saving and loading our ``LightSwitch`` along with its state-machine state can now +be accomplished as follows: + +.. code-block:: python + + >>> switch1 = LightSwitch() + >>> switch1.query_power() + False + >>> switch1.flip() + [] + >>> switch1.query_power() + True + >>> blob = switch1.save() + >>> switch2 = LightSwitch.from_blob(blob) + >>> switch2.query_power() + True + +More comprehensive (tested, working) examples are present in ``docs/examples``. + +Go forth and machine all the state! + + diff --git a/WebCrawler/venv/Lib/site-packages/Automat-20.2.0.dist-info/RECORD b/WebCrawler/venv/Lib/site-packages/Automat-20.2.0.dist-info/RECORD new file mode 100644 index 0000000..9fbad8d --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/Automat-20.2.0.dist-info/RECORD @@ -0,0 +1,32 @@ +../../Scripts/automat-visualize.exe,sha256=XoDTH-TEcSL4RPmf50yCIuDk1Pq8BYAHi8NTPZDgj54,106364 +Automat-20.2.0.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4 +Automat-20.2.0.dist-info/LICENSE,sha256=siATAWeNCpN9k4VDgnyhNgcS6zTiPejuPzv_-9TA43Y,1053 +Automat-20.2.0.dist-info/METADATA,sha256=RlZub3EgNMk1YDCNYTDJ80ksrBRzPjgzVnYsLrS44nU,17919 +Automat-20.2.0.dist-info/RECORD,, +Automat-20.2.0.dist-info/WHEEL,sha256=CihQvCnsGZQBGAHLEUMf0IdA4fRduS_NBUTMgCTtvPM,110 +Automat-20.2.0.dist-info/entry_points.txt,sha256=i4tDM5qwy3v1KIN7ETS2XRVcHkThhkq7ZFTPuyP6BpA,63 +Automat-20.2.0.dist-info/top_level.txt,sha256=vg4zAOyhP_3YCmpKZLNgFw1uMF3lC_b6TKsdz7jBSpI,8 +automat/__init__.py,sha256=ec8PILBwt35xyzsstU9kx8p5ADi6KX9d1rBLZsocN_Y,169 +automat/__pycache__/__init__.cpython-39.pyc,, +automat/__pycache__/_core.cpython-39.pyc,, +automat/__pycache__/_discover.cpython-39.pyc,, +automat/__pycache__/_introspection.cpython-39.pyc,, +automat/__pycache__/_methodical.cpython-39.pyc,, +automat/__pycache__/_visualize.cpython-39.pyc,, +automat/_core.py,sha256=IEtZHq3wsBZPX_VfMHMFy_uNjx1kfE11Qq-nmIXZe28,4819 +automat/_discover.py,sha256=ye7NHLZkrwYsPmBpTIyJuJ-VRCmwkOUpA0is-A81z04,4367 +automat/_introspection.py,sha256=i5UEGdj8lp2BnHnGeAyIwEyoN2gU1nXYg-ZPNX7oBbM,1274 +automat/_methodical.py,sha256=tvIQLMzMM1d6h_jUCQOM9zPJoGYYQZg0J5IkRszgjB0,15930 +automat/_test/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +automat/_test/__pycache__/__init__.cpython-39.pyc,, +automat/_test/__pycache__/test_core.cpython-39.pyc,, +automat/_test/__pycache__/test_discover.cpython-39.pyc,, +automat/_test/__pycache__/test_methodical.cpython-39.pyc,, +automat/_test/__pycache__/test_trace.cpython-39.pyc,, +automat/_test/__pycache__/test_visualize.cpython-39.pyc,, +automat/_test/test_core.py,sha256=CDmGBQNi9Pr7ZktfBcOhBvwI7wjLLYRtWZ0P5baXONY,2833 +automat/_test/test_discover.py,sha256=O9ndAdRAC8uO0uDhioz3e45EsJCF19jQZESj0RBC7ZM,21846 +automat/_test/test_methodical.py,sha256=t1CAKtT1fs-FoKMQxXl4ky6_6pgpG7lGDbyQrb_vIeo,18856 +automat/_test/test_trace.py,sha256=Mx1B8QgaE7QFk6blTie2j-Vx95hTV-zySnlxLalt8ek,3279 +automat/_test/test_visualize.py,sha256=8ErNYxovTiDyZkYkoP1BcyEazU_s0YQ3NHdfH9OihAg,13744 +automat/_visualize.py,sha256=jY8HkzaGdMoXB7LavvaneW4GdtBN6PRShl7-4OXDvss,6335 diff --git a/WebCrawler/venv/Lib/site-packages/Automat-20.2.0.dist-info/WHEEL b/WebCrawler/venv/Lib/site-packages/Automat-20.2.0.dist-info/WHEEL new file mode 100644 index 0000000..dea0e20 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/Automat-20.2.0.dist-info/WHEEL @@ -0,0 +1,6 @@ +Wheel-Version: 1.0 +Generator: bdist_wheel (0.32.2) +Root-Is-Purelib: true +Tag: py2-none-any +Tag: py3-none-any + diff --git a/WebCrawler/venv/Lib/site-packages/Automat-20.2.0.dist-info/entry_points.txt b/WebCrawler/venv/Lib/site-packages/Automat-20.2.0.dist-info/entry_points.txt new file mode 100644 index 0000000..d793199 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/Automat-20.2.0.dist-info/entry_points.txt @@ -0,0 +1,3 @@ +[console_scripts] +automat-visualize = automat._visualize:tool + diff --git a/WebCrawler/venv/Lib/site-packages/Automat-20.2.0.dist-info/top_level.txt b/WebCrawler/venv/Lib/site-packages/Automat-20.2.0.dist-info/top_level.txt new file mode 100644 index 0000000..b69387b --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/Automat-20.2.0.dist-info/top_level.txt @@ -0,0 +1 @@ +automat diff --git a/WebCrawler/venv/Lib/site-packages/PyHamcrest-2.0.2.dist-info/INSTALLER b/WebCrawler/venv/Lib/site-packages/PyHamcrest-2.0.2.dist-info/INSTALLER new file mode 100644 index 0000000..a1b589e --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/PyHamcrest-2.0.2.dist-info/INSTALLER @@ -0,0 +1 @@ +pip diff --git a/WebCrawler/venv/Lib/site-packages/PyHamcrest-2.0.2.dist-info/LICENSE.txt b/WebCrawler/venv/Lib/site-packages/PyHamcrest-2.0.2.dist-info/LICENSE.txt new file mode 100644 index 0000000..fe93ce1 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/PyHamcrest-2.0.2.dist-info/LICENSE.txt @@ -0,0 +1,27 @@ +BSD License + +Copyright 2020 hamcrest.org +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +Redistributions of source code must retain the above copyright notice, this list of +conditions and the following disclaimer. Redistributions in binary form must reproduce +the above copyright notice, this list of conditions and the following disclaimer in +the documentation and/or other materials provided with the distribution. + +Neither the name of Hamcrest nor the names of its contributors may be used to endorse +or promote products derived from this software without specific prior written +permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES +OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT +SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED +TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR +BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY +WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH +DAMAGE. diff --git a/WebCrawler/venv/Lib/site-packages/PyHamcrest-2.0.2.dist-info/METADATA b/WebCrawler/venv/Lib/site-packages/PyHamcrest-2.0.2.dist-info/METADATA new file mode 100644 index 0000000..78c096b --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/PyHamcrest-2.0.2.dist-info/METADATA @@ -0,0 +1,387 @@ +Metadata-Version: 2.1 +Name: PyHamcrest +Version: 2.0.2 +Summary: Hamcrest framework for matcher objects +Home-page: https://github.com/hamcrest/PyHamcrest +Author: Chris Rose +Author-email: offline@offby1.net +License: New BSD +Download-URL: http://pypi.python.org/packages/source/P/PyHamcrest/PyHamcrest-2.0.2.tar.gz +Keywords: hamcrest matchers pyunit unit test testing unittest unittesting +Platform: All +Classifier: Development Status :: 5 - Production/Stable +Classifier: Environment :: Console +Classifier: Intended Audience :: Developers +Classifier: License :: OSI Approved :: BSD License +Classifier: Natural Language :: English +Classifier: Operating System :: OS Independent +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3.5 +Classifier: Programming Language :: Python :: 3.6 +Classifier: Programming Language :: Python :: 3.7 +Classifier: Programming Language :: Python :: 3.8 +Classifier: Programming Language :: Python :: Implementation :: CPython +Classifier: Programming Language :: Python :: Implementation :: Jython +Classifier: Programming Language :: Python :: Implementation :: PyPy +Classifier: Topic :: Software Development +Classifier: Topic :: Software Development :: Quality Assurance +Classifier: Topic :: Software Development :: Testing +Provides: hamcrest +Requires-Python: >=3.5 + +PyHamcrest +========== + +| |docs| |travis| |coveralls| |landscape| |scrutinizer| +| |version| |downloads| |wheel| |supported-versions| |supported-implementations| +| |GitHub forks| |GitHub stars| |GitHub watchers| |GitHub contributors| |Lines of Code| +| |GitHub issues| |GitHub issues-closed| |GitHub pull-requests| |GitHub pull-requests closed| + +.. |docs| image:: https://readthedocs.org/projects/pyhamcrest/badge/ + :target: https://pyhamcrest.readthedocs.org/ + :alt: Documentation Status + +.. |travis| image:: http://img.shields.io/travis/hamcrest/PyHamcrest/master.svg + :alt: Travis-CI Build Status + :target: https://travis-ci.org/hamcrest/PyHamcrest + +.. |appveyor| image:: https://ci.appveyor.com/api/projects/status/github/hamcrest/PyHamcrest?branch=master&svg=true + :alt: AppVeyor Build Status + :target: https://ci.appveyor.com/project/hamcrest/PyHamcrest + +.. |coveralls| image:: http://img.shields.io/coveralls/hamcrest/PyHamcrest/master.svg?style=flat + :alt: Coverage Status + :target: https://coveralls.io/r/hamcrest/PyHamcrest + +.. |landscape| image:: https://landscape.io/github/hamcrest/PyHamcrest/master/landscape.svg?style=flat + :target: https://landscape.io/github/hamcrest/PyHamcrest/master + :alt: Code Quality Status + +.. |version| image:: http://img.shields.io/pypi/v/PyHamcrest.svg?style=flat + :alt: PyPI Package latest release + :target: https://pypi.python.org/pypi/PyHamcrest + +.. |downloads| image:: http://img.shields.io/pypi/dm/PyHamcrest.svg?style=flat + :alt: PyPI Package monthly downloads + :target: https://pypi.python.org/pypi/PyHamcrest + +.. |wheel| image:: https://pypip.in/wheel/PyHamcrest/badge.svg?style=flat + :alt: PyPI Wheel + :target: https://pypi.python.org/pypi/PyHamcrest + +.. |supported-versions| image:: https://pypip.in/py_versions/PyHamcrest/badge.svg?style=flat + :alt: Supported versions + :target: https://pypi.python.org/pypi/PyHamcrest + +.. |GitHub forks| image:: https://img.shields.io/github/forks/hamcrest/PyHamcrest.svg?label=Fork&logo=github + :alt: GitHub forks + :target: https://github.com/hamcrest/PyHamcrest/network/members + +.. |GitHub stars| image:: https://img.shields.io/github/stars/hamcrest/PyHamcrest.svg?label=Star&logo=github + :alt: GitHub stars + :target: https://github.com/hamcrest/PyHamcrest/stargazers/ + +.. |GitHub watchers| image:: https://img.shields.io/github/watchers/hamcrest/PyHamcrest.svg?label=Watch&logo=github + :alt: GitHub watchers + :target: https://github.com/hamcrest/PyHamcrest/watchers/ + +.. |GitHub contributors| image:: https://img.shields.io/github/contributors/hamcrest/PyHamcrest.svg?logo=github + :alt: GitHub contributors + :target: https://github.com/hamcrest/PyHamcrest/graphs/contributors/ + +.. |GitHub issues| image:: https://img.shields.io/github/issues/hamcrest/PyHamcrest.svg?logo=github + :alt: GitHub issues + :target: https://github.com/hamcrest/PyHamcrest/issues/ + +.. |GitHub issues-closed| image:: https://img.shields.io/github/issues-closed/hamcrest/PyHamcrest.svg?logo=github + :alt: GitHub issues-closed + :target: https://github.com/hamcrest/PyHamcrest/issues?q=is%3Aissue+is%3Aclosed + +.. |GitHub pull-requests| image:: https://img.shields.io/github/issues-pr/hamcrest/PyHamcrest.svg?logo=github + :alt: GitHub pull-requests + :target: https://github.com/hamcrest/PyHamcrest/pulls + +.. |GitHub pull-requests closed| image:: https://img.shields.io/github/issues-pr-closed/hamcrest/PyHamcrest.svg?logo=github + :alt: GitHub pull-requests closed + :target: https://github.com/hamcrest/PyHamcrest/pulls?utf8=%E2%9C%93&q=is%3Apr+is%3Aclosed + +.. |Lines of Code| image:: https://tokei.rs/b1/github/hamcrest/PyHamcrest + :alt: Lines of Code + :target: https://github.com/hamcrest/PyHamcrest + +.. |supported-implementations| image:: https://pypip.in/implementation/PyHamcrest/badge.svg?style=flat + :alt: Supported implementations + :target: https://pypi.python.org/pypi/PyHamcrest + +.. |scrutinizer| image:: https://img.shields.io/scrutinizer/g/hamcrest/PyHamcrest/master.svg?style=flat + :alt: Scrtinizer Status + :target: https://scrutinizer-ci.com/g/hamcrest/PyHamcrest/ + + +Introduction +============ + +PyHamcrest is a framework for writing matcher objects, allowing you to +declaratively define "match" rules. There are a number of situations where +matchers are invaluable, such as UI validation, or data filtering, but it is in +the area of writing flexible tests that matchers are most commonly used. This +tutorial shows you how to use PyHamcrest for unit testing. + +When writing tests it is sometimes difficult to get the balance right between +overspecifying the test (and making it brittle to changes), and not specifying +enough (making the test less valuable since it continues to pass even when the +thing being tested is broken). Having a tool that allows you to pick out +precisely the aspect under test and describe the values it should have, to a +controlled level of precision, helps greatly in writing tests that are "just +right." Such tests fail when the behavior of the aspect under test deviates +from the expected behavior, yet continue to pass when minor, unrelated changes +to the behaviour are made. + +Installation +============ + +Hamcrest can be installed using the usual Python packaging tools. It depends on +distribute, but as long as you have a network connection when you install, the +installation process will take care of that for you. + +My first PyHamcrest test +======================== + +We'll start by writing a very simple PyUnit test, but instead of using PyUnit's +``assertEqual`` method, we'll use PyHamcrest's ``assert_that`` construct and +the standard set of matchers: + +.. code:: python + + from hamcrest import * + import unittest + + class BiscuitTest(unittest.TestCase): + def testEquals(self): + theBiscuit = Biscuit('Ginger') + myBiscuit = Biscuit('Ginger') + assert_that(theBiscuit, equal_to(myBiscuit)) + + if __name__ == '__main__': + unittest.main() + +The ``assert_that`` function is a stylized sentence for making a test +assertion. In this example, the subject of the assertion is the object +``theBiscuit``, which is the first method parameter. The second method +parameter is a matcher for ``Biscuit`` objects, here a matcher that checks one +object is equal to another using the Python ``==`` operator. The test passes +since the ``Biscuit`` class defines an ``__eq__`` method. + +If you have more than one assertion in your test you can include an identifier +for the tested value in the assertion: + +.. code:: python + + assert_that(theBiscuit.getChocolateChipCount(), equal_to(10), 'chocolate chips') + assert_that(theBiscuit.getHazelnutCount(), equal_to(3), 'hazelnuts') + +As a convenience, assert_that can also be used to verify a boolean condition: + +.. code:: python + + assert_that(theBiscuit.isCooked(), 'cooked') + +This is equivalent to the ``assert_`` method of unittest.TestCase, but because +it's a standalone function, it offers greater flexibility in test writing. + + +Predefined matchers +=================== + +PyHamcrest comes with a library of useful matchers: + +* Object + + * ``equal_to`` - match equal object + * ``has_length`` - match ``len()`` + * ``has_property`` - match value of property with given name + * ``has_properties`` - match an object that has all of the given properties. + * ``has_string`` - match ``str()`` + * ``instance_of`` - match object type + * ``none``, ``not_none`` - match ``None``, or not ``None`` + * ``same_instance`` - match same object + * ``calling, raises`` - wrap a method call and assert that it raises an exception + +* Number + + * ``close_to`` - match number close to a given value + * ``greater_than``, ``greater_than_or_equal_to``, ``less_than``, + ``less_than_or_equal_to`` - match numeric ordering + +* Text + + * ``contains_string`` - match part of a string + * ``ends_with`` - match the end of a string + * ``equal_to_ignoring_case`` - match the complete string but ignore case + * ``equal_to_ignoring_whitespace`` - match the complete string but ignore extra whitespace + * ``matches_regexp`` - match a regular expression in a string + * ``starts_with`` - match the beginning of a string + * ``string_contains_in_order`` - match parts of a string, in relative order + +* Logical + + * ``all_of`` - ``and`` together all matchers + * ``any_of`` - ``or`` together all matchers + * ``anything`` - match anything, useful in composite matchers when you don't care about a particular value + * ``is_not``, ``not_`` - negate the matcher + +* Sequence + + * ``contains`` - exactly match the entire sequence + * ``contains_inanyorder`` - match the entire sequence, but in any order + * ``has_item`` - match if given item appears in the sequence + * ``has_items`` - match if all given items appear in the sequence, in any order + * ``is_in`` - match if item appears in the given sequence + * ``only_contains`` - match if sequence's items appear in given list + * ``empty`` - match if the sequence is empty + +* Dictionary + + * ``has_entries`` - match dictionary with list of key-value pairs + * ``has_entry`` - match dictionary containing a key-value pair + * ``has_key`` - match dictionary with a key + * ``has_value`` - match dictionary with a value + +* Decorator + + * ``calling`` - wrap a callable in a deferred object, for subsequent matching on calling behaviour + * ``raises`` - Ensure that a deferred callable raises as expected + * ``described_as`` - give the matcher a custom failure description + * ``is_`` - decorator to improve readability - see `Syntactic sugar` below + +The arguments for many of these matchers accept not just a matching value, but +another matcher, so matchers can be composed for greater flexibility. For +example, ``only_contains(less_than(5))`` will match any sequence where every +item is less than 5. + + +Syntactic sugar +=============== + +PyHamcrest strives to make your tests as readable as possible. For example, the +``is_`` matcher is a wrapper that doesn't add any extra behavior to the +underlying matcher. The following assertions are all equivalent: + +.. code:: python + + assert_that(theBiscuit, equal_to(myBiscuit)) + assert_that(theBiscuit, is_(equal_to(myBiscuit))) + assert_that(theBiscuit, is_(myBiscuit)) + +The last form is allowed since ``is_(value)`` wraps most non-matcher arguments +with ``equal_to``. But if the argument is a type, it is wrapped with +``instance_of``, so the following are also equivalent: + +.. code:: python + + assert_that(theBiscuit, instance_of(Biscuit)) + assert_that(theBiscuit, is_(instance_of(Biscuit))) + assert_that(theBiscuit, is_(Biscuit)) + +*Note that PyHamcrest's ``is_`` matcher is unrelated to Python's ``is`` +operator. The matcher for object identity is ``same_instance``.* + + +Writing custom matchers +======================= + +PyHamcrest comes bundled with lots of useful matchers, but you'll probably find +that you need to create your own from time to time to fit your testing needs. +This commonly occurs when you find a fragment of code that tests the same set +of properties over and over again (and in different tests), and you want to +bundle the fragment into a single assertion. By writing your own matcher you'll +eliminate code duplication and make your tests more readable! + +Let's write our own matcher for testing if a calendar date falls on a Saturday. +This is the test we want to write: + +.. code:: python + + def testDateIsOnASaturday(self): + d = datetime.date(2008, 4, 26) + assert_that(d, is_(on_a_saturday())) + +And here's the implementation: + +.. code:: python + + from hamcrest.core.base_matcher import BaseMatcher + from hamcrest.core.helpers.hasmethod import hasmethod + + class IsGivenDayOfWeek(BaseMatcher): + + def __init__(self, day): + self.day = day # Monday is 0, Sunday is 6 + + def _matches(self, item): + if not hasmethod(item, 'weekday'): + return False + return item.weekday() == self.day + + def describe_to(self, description): + day_as_string = ['Monday', 'Tuesday', 'Wednesday', 'Thursday', + 'Friday', 'Saturday', 'Sunday'] + description.append_text('calendar date falling on ') \ + .append_text(day_as_string[self.day]) + + def on_a_saturday(): + return IsGivenDayOfWeek(5) + +For our Matcher implementation we implement the ``_matches`` method - which +calls the ``weekday`` method after confirming that the argument (which may not +be a date) has such a method - and the ``describe_to`` method - which is used +to produce a failure message when a test fails. Here's an example of how the +failure message looks: + +.. code:: python + + assert_that(datetime.date(2008, 4, 6), is_(on_a_saturday())) + +fails with the message:: + + AssertionError: + Expected: is calendar date falling on Saturday + got: <2008-04-06> + +Let's say this matcher is saved in a module named ``isgivendayofweek``. We +could use it in our test by importing the factory function ``on_a_saturday``: + +.. code:: python + + from hamcrest import * + import unittest + from isgivendayofweek import on_a_saturday + + class DateTest(unittest.TestCase): + def testDateIsOnASaturday(self): + d = datetime.date(2008, 4, 26) + assert_that(d, is_(on_a_saturday())) + + if __name__ == '__main__': + unittest.main() + +Even though the ``on_a_saturday`` function creates a new matcher each time it +is called, you should not assume this is the only usage pattern for your +matcher. Therefore you should make sure your matcher is stateless, so a single +instance can be reused between matches. + + +More resources +============== + +* Documentation_ +* Package_ +* Sources_ +* Hamcrest_ + +.. _Documentation: https://pyhamcrest.readthedocs.io/ +.. _Package: http://pypi.python.org/pypi/PyHamcrest +.. _Sources: https://github.com/hamcrest/PyHamcrest +.. _Hamcrest: http://hamcrest.org + + diff --git a/WebCrawler/venv/Lib/site-packages/PyHamcrest-2.0.2.dist-info/RECORD b/WebCrawler/venv/Lib/site-packages/PyHamcrest-2.0.2.dist-info/RECORD new file mode 100644 index 0000000..87672b5 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/PyHamcrest-2.0.2.dist-info/RECORD @@ -0,0 +1,121 @@ +PyHamcrest-2.0.2.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4 +PyHamcrest-2.0.2.dist-info/LICENSE.txt,sha256=cb6ubIFRNiz-OYy9NQGRocxS09KEmYOaeDCyX1tQp_4,1468 +PyHamcrest-2.0.2.dist-info/METADATA,sha256=k9y1PGwfQq3-ZJn474IeQlLuMRBb_hfIj0AFl-DGLuU,15119 +PyHamcrest-2.0.2.dist-info/RECORD,, +PyHamcrest-2.0.2.dist-info/WHEEL,sha256=p46_5Uhzqz6AzeSosiOnxK-zmFja1i22CrQCjmYe8ec,92 +PyHamcrest-2.0.2.dist-info/top_level.txt,sha256=mRc0yPsPQSqFgWBmZBY33u-05Xtm5M4GEve4NjYdloQ,9 +hamcrest/__init__.py,sha256=Lf-BH79Mv0XxQxRQvj7jRj_kJbPmsx3muGbKDYP9QjQ,191 +hamcrest/__pycache__/__init__.cpython-39.pyc,, +hamcrest/core/__init__.py,sha256=VORMnjgpr_f_DAU2DWjMWbN7-TwApOiJYNi2tV1Zpqs,191 +hamcrest/core/__pycache__/__init__.cpython-39.pyc,, +hamcrest/core/__pycache__/assert_that.cpython-39.pyc,, +hamcrest/core/__pycache__/base_description.cpython-39.pyc,, +hamcrest/core/__pycache__/base_matcher.cpython-39.pyc,, +hamcrest/core/__pycache__/compat.cpython-39.pyc,, +hamcrest/core/__pycache__/description.cpython-39.pyc,, +hamcrest/core/__pycache__/matcher.cpython-39.pyc,, +hamcrest/core/__pycache__/selfdescribing.cpython-39.pyc,, +hamcrest/core/__pycache__/selfdescribingvalue.cpython-39.pyc,, +hamcrest/core/__pycache__/string_description.cpython-39.pyc,, +hamcrest/core/assert_that.py,sha256=jiUoGNNssb94k1euhU3AxCoDxWRYcZegH1IZeKnS81o,2821 +hamcrest/core/base_description.py,sha256=5uJ4GiJWEqFR1SD32zHAk2vUd0UO_4StVnmZ48hmot4,2028 +hamcrest/core/base_matcher.py,sha256=N6P0Z8Qx3uo8NmrjtOLH2xaS9CEMvf_QQKjnHIyiSXU,1495 +hamcrest/core/compat.py,sha256=wUi1u_nIhgUA3kFGJDZLQ2D_cRw79E1SxPDMaI4mzG0,642 +hamcrest/core/core/__init__.py,sha256=xg5Q2ugy7LsKqgx89utyUbvWGRq0N2ouEggu5njJK10,731 +hamcrest/core/core/__pycache__/__init__.cpython-39.pyc,, +hamcrest/core/core/__pycache__/allof.cpython-39.pyc,, +hamcrest/core/core/__pycache__/anyof.cpython-39.pyc,, +hamcrest/core/core/__pycache__/described_as.cpython-39.pyc,, +hamcrest/core/core/__pycache__/is_.cpython-39.pyc,, +hamcrest/core/core/__pycache__/isanything.cpython-39.pyc,, +hamcrest/core/core/__pycache__/isequal.cpython-39.pyc,, +hamcrest/core/core/__pycache__/isinstanceof.cpython-39.pyc,, +hamcrest/core/core/__pycache__/isnone.cpython-39.pyc,, +hamcrest/core/core/__pycache__/isnot.cpython-39.pyc,, +hamcrest/core/core/__pycache__/issame.cpython-39.pyc,, +hamcrest/core/core/__pycache__/raises.cpython-39.pyc,, +hamcrest/core/core/allof.py,sha256=QlKOUPTeB5VLOQBVbi1RVgpyE4F_CLL1M4fQREzCT2Q,2319 +hamcrest/core/core/anyof.py,sha256=Ik2qXhLtbwddvT6pFf1HD_6xE1qQm1Aaut9z2B0Lh34,1332 +hamcrest/core/core/described_as.py,sha256=PT38Tp4rCRFWLWWOEzLPNPr_J9HOa4AOPdzb1Qi6Uf4,1939 +hamcrest/core/core/is_.py,sha256=j4UVPaLbwKuyjSmMyJTEtYsgxaet856OU6cxQ4_EskI,3040 +hamcrest/core/core/isanything.py,sha256=KvwblH_oAslffVTwzfb4g0pZpcLBwTwh2IqN23m7JoE,975 +hamcrest/core/core/isequal.py,sha256=L8hQ9vJyxhx9CU0xoezyrAOOZclPg4lp2qWkj5oZVaE,1039 +hamcrest/core/core/isinstanceof.py,sha256=rWdTHfMjxaQJrfAbWTQ4ZVxVoBc_gL_E6hTUX4GkfYg,1261 +hamcrest/core/core/isnone.py,sha256=ttgbU_fTH-oYQ-rketRE8X-EHSSBJOeG_FsV2WZAjiA,745 +hamcrest/core/core/isnot.py,sha256=hUub3YLsPHssX7a237jA6sEx5UJTNIcyjD_1PC4h8dw,2186 +hamcrest/core/core/issame.py,sha256=tyy8LPix9RBH3l2BHQMwmPdgqN3ROPP9UExA026Iycs,1344 +hamcrest/core/core/raises.py,sha256=S7519X5nMqYw3K3RWqDroAat7J2TgBNTFuGKCvO8w6A,5384 +hamcrest/core/description.py,sha256=w_t6piV7jdeNUi5BFNvtepFb4uOSeo8umQ_ltaRoUpo,1545 +hamcrest/core/helpers/__init__.py,sha256=mPsycYI18LoGawOo9BfETa7yKtnM-fDjFOr43BIevUg,146 +hamcrest/core/helpers/__pycache__/__init__.cpython-39.pyc,, +hamcrest/core/helpers/__pycache__/hasmethod.cpython-39.pyc,, +hamcrest/core/helpers/__pycache__/ismock.cpython-39.pyc,, +hamcrest/core/helpers/__pycache__/wrap_matcher.cpython-39.pyc,, +hamcrest/core/helpers/hasmethod.py,sha256=y2Te7m2edjLsZFH7Aadtnv81YKglTr5rYtr84O_sGgM,346 +hamcrest/core/helpers/ismock.py,sha256=nMpzjr4wnP6KBJgNG9K656z3x2bt8-TZHW2x08wpxLE,327 +hamcrest/core/helpers/wrap_matcher.py,sha256=--N9XNoByGw_L7cd66phQIukXUzRwLkVwqfw--PbY-w,748 +hamcrest/core/matcher.py,sha256=rzbXeizVYubPreI4t6LeFTzj8A1u7gVemC1iZ7wZb-M,2664 +hamcrest/core/selfdescribing.py,sha256=JEu7jzBRyZdNgUFW1nmSbu-AVTBV04DMN_lhgAQ2MWI,638 +hamcrest/core/selfdescribingvalue.py,sha256=z_HNxqgXjw3PJ-MiNQEd7Uf-UCp7_zkhY_tituiCeI0,928 +hamcrest/core/string_description.py,sha256=IWqdu3esHWVPv28P3oUs0Y8UQXCHZVpSF2rs_XOYo-s,940 +hamcrest/library/__init__.py,sha256=tf9nJwXFdyef20MSD81R1xWhjecbkeCvsttA5u7fpX0,999 +hamcrest/library/__pycache__/__init__.cpython-39.pyc,, +hamcrest/library/collection/__init__.py,sha256=eNJTFJ8O34E_PKJiQ9S9XhDwysfWK7f2AZCLACoFGwE,614 +hamcrest/library/collection/__pycache__/__init__.cpython-39.pyc,, +hamcrest/library/collection/__pycache__/is_empty.cpython-39.pyc,, +hamcrest/library/collection/__pycache__/isdict_containing.cpython-39.pyc,, +hamcrest/library/collection/__pycache__/isdict_containingentries.cpython-39.pyc,, +hamcrest/library/collection/__pycache__/isdict_containingkey.cpython-39.pyc,, +hamcrest/library/collection/__pycache__/isdict_containingvalue.cpython-39.pyc,, +hamcrest/library/collection/__pycache__/isin.cpython-39.pyc,, +hamcrest/library/collection/__pycache__/issequence_containing.cpython-39.pyc,, +hamcrest/library/collection/__pycache__/issequence_containinginanyorder.cpython-39.pyc,, +hamcrest/library/collection/__pycache__/issequence_containinginorder.cpython-39.pyc,, +hamcrest/library/collection/__pycache__/issequence_onlycontaining.cpython-39.pyc,, +hamcrest/library/collection/is_empty.py,sha256=CxteISdn1Ne75BwvM6m2A8ccfiVMZav0dBxaTrhDeFY,1076 +hamcrest/library/collection/isdict_containing.py,sha256=WrXln6YJ6Ow0cf97yL2XbNIdC81Wjj9Euor9rBxzPRM,2305 +hamcrest/library/collection/isdict_containingentries.py,sha256=NLU9s5MLJYvPX3q_QkL24XWdSohpq8LCUjiQdO3GUYs,5692 +hamcrest/library/collection/isdict_containingkey.py,sha256=vu6ZqpwpkJ_ez-_IOqiO-In7ICF9Vs8kNX1VRWdCtmk,1829 +hamcrest/library/collection/isdict_containingvalue.py,sha256=S08RBkdrzYW1URsdj3tWaxya0uUdY3CB3CdYMktDB1M,1835 +hamcrest/library/collection/isin.py,sha256=_fNAS55mHVYZa_1AOHe1ti1oaV6gkXr7rDfCrA0ZUtE,977 +hamcrest/library/collection/issequence_containing.py,sha256=w6GSl_5jctJQDYVlWqau8sDGhzP2N0Qf_g82z3TyVIA,3432 +hamcrest/library/collection/issequence_containinginanyorder.py,sha256=kORH137ACuJCah5KHaW0aezkFsi6y9V9WBubaiQjXMk,3869 +hamcrest/library/collection/issequence_containinginorder.py,sha256=jGYYivLAWzlFTyXSFDknJUz3QWwLk7UDXFwJ96ZsRbg,3833 +hamcrest/library/collection/issequence_onlycontaining.py,sha256=r3PM88_2sY5z1898mG2CM2KlZTCVhNDFxuI2agk8JMc,1841 +hamcrest/library/integration/__init__.py,sha256=wIFPkUnh1CkcODPv-KKQ_hPMGGdkLfMnIbIgDyl1MjE,215 +hamcrest/library/integration/__pycache__/__init__.cpython-39.pyc,, +hamcrest/library/integration/__pycache__/match_equality.cpython-39.pyc,, +hamcrest/library/integration/match_equality.py,sha256=rTa1Ae_yg1aeb8PWW1ySFzt9BPgYpd6TdLR9o4AGsko,1323 +hamcrest/library/number/__init__.py,sha256=qEsI3wwSIZtXQfbbVzvYYimdR31deOFqDSd8hiLc-j0,317 +hamcrest/library/number/__pycache__/__init__.cpython-39.pyc,, +hamcrest/library/number/__pycache__/iscloseto.cpython-39.pyc,, +hamcrest/library/number/__pycache__/ordering_comparison.cpython-39.pyc,, +hamcrest/library/number/iscloseto.py,sha256=e1_-r0YZGcjf6moA1x8_uvMFmSHYwI09Ru1Lx9h4ZzQ,2742 +hamcrest/library/number/ordering_comparison.py,sha256=QRY53Q68bTLwq8EnorvNa4A7WrKdXw5kHi-mzcMzsB0,1939 +hamcrest/library/object/__init__.py,sha256=GOROM3mNGZsO09KMNtkT9lUVDcKwuzi4oQa5nqYHD6g,280 +hamcrest/library/object/__pycache__/__init__.cpython-39.pyc,, +hamcrest/library/object/__pycache__/haslength.cpython-39.pyc,, +hamcrest/library/object/__pycache__/hasproperty.cpython-39.pyc,, +hamcrest/library/object/__pycache__/hasstring.cpython-39.pyc,, +hamcrest/library/object/haslength.py,sha256=WLPQJUbUeLvIsTom_BcRBhvTIARv0J8hAk62e49SJew,1905 +hamcrest/library/object/hasproperty.py,sha256=0YXzHXPIa56ZrNBrI4861-Y79zwAnlG7P9iUAjwgkUk,7154 +hamcrest/library/object/hasstring.py,sha256=rZTgZbrhzsy0QwkXI_MBUAr06JUzj-Um3f7TmuVh7Vc,1395 +hamcrest/library/text/__init__.py,sha256=YuMghVZsfFOVFHEAVGqso-K9GH5TA4uwQw-VLQy53XA,509 +hamcrest/library/text/__pycache__/__init__.cpython-39.pyc,, +hamcrest/library/text/__pycache__/isequal_ignoring_case.cpython-39.pyc,, +hamcrest/library/text/__pycache__/isequal_ignoring_whitespace.cpython-39.pyc,, +hamcrest/library/text/__pycache__/stringcontains.cpython-39.pyc,, +hamcrest/library/text/__pycache__/stringcontainsinorder.cpython-39.pyc,, +hamcrest/library/text/__pycache__/stringendswith.cpython-39.pyc,, +hamcrest/library/text/__pycache__/stringmatches.cpython-39.pyc,, +hamcrest/library/text/__pycache__/stringstartswith.cpython-39.pyc,, +hamcrest/library/text/__pycache__/substringmatcher.cpython-39.pyc,, +hamcrest/library/text/isequal_ignoring_case.py,sha256=131_cbPxc-OjiUMjOZo1JNaO6ElNYnR-FghO4oyEsvE,1359 +hamcrest/library/text/isequal_ignoring_whitespace.py,sha256=829ussSeFtzIp-SjDNCgUj4XL2ZFbJAANq9Fu4vCKKs,1776 +hamcrest/library/text/stringcontains.py,sha256=ix7HlqKMjHhqs4PhX-5GaNAdR9HKqHYg6YkYdKBkkTg,1036 +hamcrest/library/text/stringcontainsinorder.py,sha256=snxoa2Y2umIY4oC2O1zWrjKlpqDbYeNkf2aFxpEoTe8,1774 +hamcrest/library/text/stringendswith.py,sha256=HZ43F8uva4dCC7ZkrxHrOCnEYtGwu7nKkysS9sXTNCw,1063 +hamcrest/library/text/stringmatches.py,sha256=nq8K7WcFXCQKdH_zrXFqJa-DtltXqmb3C2yxK43hlHc,1233 +hamcrest/library/text/stringstartswith.py,sha256=lIb0PuBlEerbPlqBWdTW3LEhYVXjNTuoZKhDN8Hh20E,1090 +hamcrest/library/text/substringmatcher.py,sha256=Ji-OrvF0Br7KgMALgt-LtE1WqqzUrX2j3-JCBWx75fM,786 +hamcrest/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 diff --git a/WebCrawler/venv/Lib/site-packages/PyHamcrest-2.0.2.dist-info/WHEEL b/WebCrawler/venv/Lib/site-packages/PyHamcrest-2.0.2.dist-info/WHEEL new file mode 100644 index 0000000..3b5c403 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/PyHamcrest-2.0.2.dist-info/WHEEL @@ -0,0 +1,5 @@ +Wheel-Version: 1.0 +Generator: bdist_wheel (0.33.6) +Root-Is-Purelib: true +Tag: py3-none-any + diff --git a/WebCrawler/venv/Lib/site-packages/PyHamcrest-2.0.2.dist-info/top_level.txt b/WebCrawler/venv/Lib/site-packages/PyHamcrest-2.0.2.dist-info/top_level.txt new file mode 100644 index 0000000..c5f2ee4 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/PyHamcrest-2.0.2.dist-info/top_level.txt @@ -0,0 +1 @@ +hamcrest diff --git a/WebCrawler/venv/Lib/site-packages/__pycache__/easy_install.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/__pycache__/easy_install.cpython-39.pyc new file mode 100644 index 0000000..8abbc92 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/__pycache__/easy_install.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/__pycache__/mccabe.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/__pycache__/mccabe.cpython-39.pyc new file mode 100644 index 0000000..6a8073e Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/__pycache__/mccabe.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/__pycache__/six.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/__pycache__/six.cpython-39.pyc new file mode 100644 index 0000000..46bfa02 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/__pycache__/six.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/_cffi_backend.cp39-win_amd64.pyd b/WebCrawler/venv/Lib/site-packages/_cffi_backend.cp39-win_amd64.pyd new file mode 100644 index 0000000..8ad6008 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/_cffi_backend.cp39-win_amd64.pyd differ diff --git a/WebCrawler/venv/Lib/site-packages/astroid-2.4.2.dist-info/COPYING b/WebCrawler/venv/Lib/site-packages/astroid-2.4.2.dist-info/COPYING new file mode 100644 index 0000000..d511905 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/astroid-2.4.2.dist-info/COPYING @@ -0,0 +1,339 @@ + GNU GENERAL PUBLIC LICENSE + Version 2, June 1991 + + Copyright (C) 1989, 1991 Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +License is intended to guarantee your freedom to share and change free +software--to make sure the software is free for all its users. This +General Public License applies to most of the Free Software +Foundation's software and to any other program whose authors commit to +using it. (Some other Free Software Foundation software is covered by +the GNU Lesser General Public License instead.) You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +this service if you wish), that you receive source code or can get it +if you want it, that you can change the software or use pieces of it +in new free programs; and that you know you can do these things. + + To protect your rights, we need to make restrictions that forbid +anyone to deny you these rights or to ask you to surrender the rights. +These restrictions translate to certain responsibilities for you if you +distribute copies of the software, or if you modify it. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must give the recipients all the rights that +you have. You must make sure that they, too, receive or can get the +source code. And you must show them these terms so they know their +rights. + + We protect your rights with two steps: (1) copyright the software, and +(2) offer you this license which gives you legal permission to copy, +distribute and/or modify the software. + + Also, for each author's protection and ours, we want to make certain +that everyone understands that there is no warranty for this free +software. If the software is modified by someone else and passed on, we +want its recipients to know that what they have is not the original, so +that any problems introduced by others will not reflect on the original +authors' reputations. + + Finally, any free program is threatened constantly by software +patents. We wish to avoid the danger that redistributors of a free +program will individually obtain patent licenses, in effect making the +program proprietary. To prevent this, we have made it clear that any +patent must be licensed for everyone's free use or not licensed at all. + + The precise terms and conditions for copying, distribution and +modification follow. + + GNU GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License applies to any program or other work which contains +a notice placed by the copyright holder saying it may be distributed +under the terms of this General Public License. The "Program", below, +refers to any such program or work, and a "work based on the Program" +means either the Program or any derivative work under copyright law: +that is to say, a work containing the Program or a portion of it, +either verbatim or with modifications and/or translated into another +language. (Hereinafter, translation is included without limitation in +the term "modification".) Each licensee is addressed as "you". + +Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running the Program is not restricted, and the output from the Program +is covered only if its contents constitute a work based on the +Program (independent of having been made by running the Program). +Whether that is true depends on what the Program does. + + 1. You may copy and distribute verbatim copies of the Program's +source code as you receive it, in any medium, provided that you +conspicuously and appropriately publish on each copy an appropriate +copyright notice and disclaimer of warranty; keep intact all the +notices that refer to this License and to the absence of any warranty; +and give any other recipients of the Program a copy of this License +along with the Program. + +You may charge a fee for the physical act of transferring a copy, and +you may at your option offer warranty protection in exchange for a fee. + + 2. You may modify your copy or copies of the Program or any portion +of it, thus forming a work based on the Program, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) You must cause the modified files to carry prominent notices + stating that you changed the files and the date of any change. + + b) You must cause any work that you distribute or publish, that in + whole or in part contains or is derived from the Program or any + part thereof, to be licensed as a whole at no charge to all third + parties under the terms of this License. + + c) If the modified program normally reads commands interactively + when run, you must cause it, when started running for such + interactive use in the most ordinary way, to print or display an + announcement including an appropriate copyright notice and a + notice that there is no warranty (or else, saying that you provide + a warranty) and that users may redistribute the program under + these conditions, and telling the user how to view a copy of this + License. (Exception: if the Program itself is interactive but + does not normally print such an announcement, your work based on + the Program is not required to print an announcement.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Program, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Program, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Program. + +In addition, mere aggregation of another work not based on the Program +with the Program (or with a work based on the Program) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may copy and distribute the Program (or a work based on it, +under Section 2) in object code or executable form under the terms of +Sections 1 and 2 above provided that you also do one of the following: + + a) Accompany it with the complete corresponding machine-readable + source code, which must be distributed under the terms of Sections + 1 and 2 above on a medium customarily used for software interchange; or, + + b) Accompany it with a written offer, valid for at least three + years, to give any third party, for a charge no more than your + cost of physically performing source distribution, a complete + machine-readable copy of the corresponding source code, to be + distributed under the terms of Sections 1 and 2 above on a medium + customarily used for software interchange; or, + + c) Accompany it with the information you received as to the offer + to distribute corresponding source code. (This alternative is + allowed only for noncommercial distribution and only if you + received the program in object code or executable form with such + an offer, in accord with Subsection b above.) + +The source code for a work means the preferred form of the work for +making modifications to it. For an executable work, complete source +code means all the source code for all modules it contains, plus any +associated interface definition files, plus the scripts used to +control compilation and installation of the executable. However, as a +special exception, the source code distributed need not include +anything that is normally distributed (in either source or binary +form) with the major components (compiler, kernel, and so on) of the +operating system on which the executable runs, unless that component +itself accompanies the executable. + +If distribution of executable or object code is made by offering +access to copy from a designated place, then offering equivalent +access to copy the source code from the same place counts as +distribution of the source code, even though third parties are not +compelled to copy the source along with the object code. + + 4. You may not copy, modify, sublicense, or distribute the Program +except as expressly provided under this License. Any attempt +otherwise to copy, modify, sublicense or distribute the Program is +void, and will automatically terminate your rights under this License. +However, parties who have received copies, or rights, from you under +this License will not have their licenses terminated so long as such +parties remain in full compliance. + + 5. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Program or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Program (or any work based on the +Program), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Program or works based on it. + + 6. Each time you redistribute the Program (or any work based on the +Program), the recipient automatically receives a license from the +original licensor to copy, distribute or modify the Program subject to +these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties to +this License. + + 7. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Program at all. For example, if a patent +license would not permit royalty-free redistribution of the Program by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Program. + +If any portion of this section is held invalid or unenforceable under +any particular circumstance, the balance of the section is intended to +apply and the section as a whole is intended to apply in other +circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system, which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 8. If the distribution and/or use of the Program is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Program under this License +may add an explicit geographical distribution limitation excluding +those countries, so that distribution is permitted only in or among +countries not thus excluded. In such case, this License incorporates +the limitation as if written in the body of this License. + + 9. The Free Software Foundation may publish revised and/or new versions +of the General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + +Each version is given a distinguishing version number. If the Program +specifies a version number of this License which applies to it and "any +later version", you have the option of following the terms and conditions +either of that version or of any later version published by the Free +Software Foundation. If the Program does not specify a version number of +this License, you may choose any version ever published by the Free Software +Foundation. + + 10. If you wish to incorporate parts of the Program into other free +programs whose distribution conditions are different, write to the author +to ask for permission. For software which is copyrighted by the Free +Software Foundation, write to the Free Software Foundation; we sometimes +make exceptions for this. Our decision will be guided by the two goals +of preserving the free status of all derivatives of our free software and +of promoting the sharing and reuse of software generally. + + NO WARRANTY + + 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY +FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN +OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES +PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED +OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS +TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE +PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, +REPAIR OR CORRECTION. + + 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR +REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, +INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING +OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED +TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY +YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER +PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +Also add information on how to contact you by electronic and paper mail. + +If the program is interactive, make it output a short notice like this +when it starts in an interactive mode: + + Gnomovision version 69, Copyright (C) year name of author + Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, the commands you use may +be called something other than `show w' and `show c'; they could even be +mouse-clicks or menu items--whatever suits your program. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the program, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the program + `Gnomovision' (which makes passes at compilers) written by James Hacker. + + , 1 April 1989 + Ty Coon, President of Vice + +This General Public License does not permit incorporating your program into +proprietary programs. If your program is a subroutine library, you may +consider it more useful to permit linking proprietary applications with the +library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. diff --git a/WebCrawler/venv/Lib/site-packages/astroid-2.4.2.dist-info/COPYING.LESSER b/WebCrawler/venv/Lib/site-packages/astroid-2.4.2.dist-info/COPYING.LESSER new file mode 100644 index 0000000..2d2d780 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/astroid-2.4.2.dist-info/COPYING.LESSER @@ -0,0 +1,510 @@ + + GNU LESSER GENERAL PUBLIC LICENSE + Version 2.1, February 1999 + + Copyright (C) 1991, 1999 Free Software Foundation, Inc. + 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + +[This is the first released version of the Lesser GPL. It also counts + as the successor of the GNU Library Public License, version 2, hence + the version number 2.1.] + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +Licenses are intended to guarantee your freedom to share and change +free software--to make sure the software is free for all its users. + + This license, the Lesser General Public License, applies to some +specially designated software packages--typically libraries--of the +Free Software Foundation and other authors who decide to use it. You +can use it too, but we suggest you first think carefully about whether +this license or the ordinary General Public License is the better +strategy to use in any particular case, based on the explanations +below. + + When we speak of free software, we are referring to freedom of use, +not price. Our General Public Licenses are designed to make sure that +you have the freedom to distribute copies of free software (and charge +for this service if you wish); that you receive source code or can get +it if you want it; that you can change the software and use pieces of +it in new free programs; and that you are informed that you can do +these things. + + To protect your rights, we need to make restrictions that forbid +distributors to deny you these rights or to ask you to surrender these +rights. These restrictions translate to certain responsibilities for +you if you distribute copies of the library or if you modify it. + + For example, if you distribute copies of the library, whether gratis +or for a fee, you must give the recipients all the rights that we gave +you. You must make sure that they, too, receive or can get the source +code. If you link other code with the library, you must provide +complete object files to the recipients, so that they can relink them +with the library after making changes to the library and recompiling +it. And you must show them these terms so they know their rights. + + We protect your rights with a two-step method: (1) we copyright the +library, and (2) we offer you this license, which gives you legal +permission to copy, distribute and/or modify the library. + + To protect each distributor, we want to make it very clear that +there is no warranty for the free library. Also, if the library is +modified by someone else and passed on, the recipients should know +that what they have is not the original version, so that the original +author's reputation will not be affected by problems that might be +introduced by others. + + Finally, software patents pose a constant threat to the existence of +any free program. We wish to make sure that a company cannot +effectively restrict the users of a free program by obtaining a +restrictive license from a patent holder. Therefore, we insist that +any patent license obtained for a version of the library must be +consistent with the full freedom of use specified in this license. + + Most GNU software, including some libraries, is covered by the +ordinary GNU General Public License. This license, the GNU Lesser +General Public License, applies to certain designated libraries, and +is quite different from the ordinary General Public License. We use +this license for certain libraries in order to permit linking those +libraries into non-free programs. + + When a program is linked with a library, whether statically or using +a shared library, the combination of the two is legally speaking a +combined work, a derivative of the original library. The ordinary +General Public License therefore permits such linking only if the +entire combination fits its criteria of freedom. The Lesser General +Public License permits more lax criteria for linking other code with +the library. + + We call this license the "Lesser" General Public License because it +does Less to protect the user's freedom than the ordinary General +Public License. It also provides other free software developers Less +of an advantage over competing non-free programs. These disadvantages +are the reason we use the ordinary General Public License for many +libraries. However, the Lesser license provides advantages in certain +special circumstances. + + For example, on rare occasions, there may be a special need to +encourage the widest possible use of a certain library, so that it +becomes a de-facto standard. To achieve this, non-free programs must +be allowed to use the library. A more frequent case is that a free +library does the same job as widely used non-free libraries. In this +case, there is little to gain by limiting the free library to free +software only, so we use the Lesser General Public License. + + In other cases, permission to use a particular library in non-free +programs enables a greater number of people to use a large body of +free software. For example, permission to use the GNU C Library in +non-free programs enables many more people to use the whole GNU +operating system, as well as its variant, the GNU/Linux operating +system. + + Although the Lesser General Public License is Less protective of the +users' freedom, it does ensure that the user of a program that is +linked with the Library has the freedom and the wherewithal to run +that program using a modified version of the Library. + + The precise terms and conditions for copying, distribution and +modification follow. Pay close attention to the difference between a +"work based on the library" and a "work that uses the library". The +former contains code derived from the library, whereas the latter must +be combined with the library in order to run. + + GNU LESSER GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License Agreement applies to any software library or other +program which contains a notice placed by the copyright holder or +other authorized party saying it may be distributed under the terms of +this Lesser General Public License (also called "this License"). +Each licensee is addressed as "you". + + A "library" means a collection of software functions and/or data +prepared so as to be conveniently linked with application programs +(which use some of those functions and data) to form executables. + + The "Library", below, refers to any such software library or work +which has been distributed under these terms. A "work based on the +Library" means either the Library or any derivative work under +copyright law: that is to say, a work containing the Library or a +portion of it, either verbatim or with modifications and/or translated +straightforwardly into another language. (Hereinafter, translation is +included without limitation in the term "modification".) + + "Source code" for a work means the preferred form of the work for +making modifications to it. For a library, complete source code means +all the source code for all modules it contains, plus any associated +interface definition files, plus the scripts used to control +compilation and installation of the library. + + Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running a program using the Library is not restricted, and output from +such a program is covered only if its contents constitute a work based +on the Library (independent of the use of the Library in a tool for +writing it). Whether that is true depends on what the Library does +and what the program that uses the Library does. + + 1. You may copy and distribute verbatim copies of the Library's +complete source code as you receive it, in any medium, provided that +you conspicuously and appropriately publish on each copy an +appropriate copyright notice and disclaimer of warranty; keep intact +all the notices that refer to this License and to the absence of any +warranty; and distribute a copy of this License along with the +Library. + + You may charge a fee for the physical act of transferring a copy, +and you may at your option offer warranty protection in exchange for a +fee. + + 2. You may modify your copy or copies of the Library or any portion +of it, thus forming a work based on the Library, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) The modified work must itself be a software library. + + b) You must cause the files modified to carry prominent notices + stating that you changed the files and the date of any change. + + c) You must cause the whole of the work to be licensed at no + charge to all third parties under the terms of this License. + + d) If a facility in the modified Library refers to a function or a + table of data to be supplied by an application program that uses + the facility, other than as an argument passed when the facility + is invoked, then you must make a good faith effort to ensure that, + in the event an application does not supply such function or + table, the facility still operates, and performs whatever part of + its purpose remains meaningful. + + (For example, a function in a library to compute square roots has + a purpose that is entirely well-defined independent of the + application. Therefore, Subsection 2d requires that any + application-supplied function or table used by this function must + be optional: if the application does not supply it, the square + root function must still compute square roots.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Library, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Library, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote +it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Library. + +In addition, mere aggregation of another work not based on the Library +with the Library (or with a work based on the Library) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may opt to apply the terms of the ordinary GNU General Public +License instead of this License to a given copy of the Library. To do +this, you must alter all the notices that refer to this License, so +that they refer to the ordinary GNU General Public License, version 2, +instead of to this License. (If a newer version than version 2 of the +ordinary GNU General Public License has appeared, then you can specify +that version instead if you wish.) Do not make any other change in +these notices. + + Once this change is made in a given copy, it is irreversible for +that copy, so the ordinary GNU General Public License applies to all +subsequent copies and derivative works made from that copy. + + This option is useful when you wish to copy part of the code of +the Library into a program that is not a library. + + 4. You may copy and distribute the Library (or a portion or +derivative of it, under Section 2) in object code or executable form +under the terms of Sections 1 and 2 above provided that you accompany +it with the complete corresponding machine-readable source code, which +must be distributed under the terms of Sections 1 and 2 above on a +medium customarily used for software interchange. + + If distribution of object code is made by offering access to copy +from a designated place, then offering equivalent access to copy the +source code from the same place satisfies the requirement to +distribute the source code, even though third parties are not +compelled to copy the source along with the object code. + + 5. A program that contains no derivative of any portion of the +Library, but is designed to work with the Library by being compiled or +linked with it, is called a "work that uses the Library". Such a +work, in isolation, is not a derivative work of the Library, and +therefore falls outside the scope of this License. + + However, linking a "work that uses the Library" with the Library +creates an executable that is a derivative of the Library (because it +contains portions of the Library), rather than a "work that uses the +library". The executable is therefore covered by this License. +Section 6 states terms for distribution of such executables. + + When a "work that uses the Library" uses material from a header file +that is part of the Library, the object code for the work may be a +derivative work of the Library even though the source code is not. +Whether this is true is especially significant if the work can be +linked without the Library, or if the work is itself a library. The +threshold for this to be true is not precisely defined by law. + + If such an object file uses only numerical parameters, data +structure layouts and accessors, and small macros and small inline +functions (ten lines or less in length), then the use of the object +file is unrestricted, regardless of whether it is legally a derivative +work. (Executables containing this object code plus portions of the +Library will still fall under Section 6.) + + Otherwise, if the work is a derivative of the Library, you may +distribute the object code for the work under the terms of Section 6. +Any executables containing that work also fall under Section 6, +whether or not they are linked directly with the Library itself. + + 6. As an exception to the Sections above, you may also combine or +link a "work that uses the Library" with the Library to produce a +work containing portions of the Library, and distribute that work +under terms of your choice, provided that the terms permit +modification of the work for the customer's own use and reverse +engineering for debugging such modifications. + + You must give prominent notice with each copy of the work that the +Library is used in it and that the Library and its use are covered by +this License. You must supply a copy of this License. If the work +during execution displays copyright notices, you must include the +copyright notice for the Library among them, as well as a reference +directing the user to the copy of this License. Also, you must do one +of these things: + + a) Accompany the work with the complete corresponding + machine-readable source code for the Library including whatever + changes were used in the work (which must be distributed under + Sections 1 and 2 above); and, if the work is an executable linked + with the Library, with the complete machine-readable "work that + uses the Library", as object code and/or source code, so that the + user can modify the Library and then relink to produce a modified + executable containing the modified Library. (It is understood + that the user who changes the contents of definitions files in the + Library will not necessarily be able to recompile the application + to use the modified definitions.) + + b) Use a suitable shared library mechanism for linking with the + Library. A suitable mechanism is one that (1) uses at run time a + copy of the library already present on the user's computer system, + rather than copying library functions into the executable, and (2) + will operate properly with a modified version of the library, if + the user installs one, as long as the modified version is + interface-compatible with the version that the work was made with. + + c) Accompany the work with a written offer, valid for at least + three years, to give the same user the materials specified in + Subsection 6a, above, for a charge no more than the cost of + performing this distribution. + + d) If distribution of the work is made by offering access to copy + from a designated place, offer equivalent access to copy the above + specified materials from the same place. + + e) Verify that the user has already received a copy of these + materials or that you have already sent this user a copy. + + For an executable, the required form of the "work that uses the +Library" must include any data and utility programs needed for +reproducing the executable from it. However, as a special exception, +the materials to be distributed need not include anything that is +normally distributed (in either source or binary form) with the major +components (compiler, kernel, and so on) of the operating system on +which the executable runs, unless that component itself accompanies +the executable. + + It may happen that this requirement contradicts the license +restrictions of other proprietary libraries that do not normally +accompany the operating system. Such a contradiction means you cannot +use both them and the Library together in an executable that you +distribute. + + 7. You may place library facilities that are a work based on the +Library side-by-side in a single library together with other library +facilities not covered by this License, and distribute such a combined +library, provided that the separate distribution of the work based on +the Library and of the other library facilities is otherwise +permitted, and provided that you do these two things: + + a) Accompany the combined library with a copy of the same work + based on the Library, uncombined with any other library + facilities. This must be distributed under the terms of the + Sections above. + + b) Give prominent notice with the combined library of the fact + that part of it is a work based on the Library, and explaining + where to find the accompanying uncombined form of the same work. + + 8. You may not copy, modify, sublicense, link with, or distribute +the Library except as expressly provided under this License. Any +attempt otherwise to copy, modify, sublicense, link with, or +distribute the Library is void, and will automatically terminate your +rights under this License. However, parties who have received copies, +or rights, from you under this License will not have their licenses +terminated so long as such parties remain in full compliance. + + 9. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Library or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Library (or any work based on the +Library), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Library or works based on it. + + 10. Each time you redistribute the Library (or any work based on the +Library), the recipient automatically receives a license from the +original licensor to copy, distribute, link with or modify the Library +subject to these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties with +this License. + + 11. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Library at all. For example, if a patent +license would not permit royalty-free redistribution of the Library by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Library. + +If any portion of this section is held invalid or unenforceable under +any particular circumstance, the balance of the section is intended to +apply, and the section as a whole is intended to apply in other +circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 12. If the distribution and/or use of the Library is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Library under this License +may add an explicit geographical distribution limitation excluding those +countries, so that distribution is permitted only in or among +countries not thus excluded. In such case, this License incorporates +the limitation as if written in the body of this License. + + 13. The Free Software Foundation may publish revised and/or new +versions of the Lesser General Public License from time to time. +Such new versions will be similar in spirit to the present version, +but may differ in detail to address new problems or concerns. + +Each version is given a distinguishing version number. If the Library +specifies a version number of this License which applies to it and +"any later version", you have the option of following the terms and +conditions either of that version or of any later version published by +the Free Software Foundation. If the Library does not specify a +license version number, you may choose any version ever published by +the Free Software Foundation. + + 14. If you wish to incorporate parts of the Library into other free +programs whose distribution conditions are incompatible with these, +write to the author to ask for permission. For software which is +copyrighted by the Free Software Foundation, write to the Free +Software Foundation; we sometimes make exceptions for this. Our +decision will be guided by the two goals of preserving the free status +of all derivatives of our free software and of promoting the sharing +and reuse of software generally. + + NO WARRANTY + + 15. BECAUSE THE LIBRARY IS LICENSED FREE OF CHARGE, THERE IS NO +WARRANTY FOR THE LIBRARY, TO THE EXTENT PERMITTED BY APPLICABLE LAW. +EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR +OTHER PARTIES PROVIDE THE LIBRARY "AS IS" WITHOUT WARRANTY OF ANY +KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE +LIBRARY IS WITH YOU. SHOULD THE LIBRARY PROVE DEFECTIVE, YOU ASSUME +THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN +WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY +AND/OR REDISTRIBUTE THE LIBRARY AS PERMITTED ABOVE, BE LIABLE TO YOU +FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR +CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE +LIBRARY (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING +RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A +FAILURE OF THE LIBRARY TO OPERATE WITH ANY OTHER SOFTWARE), EVEN IF +SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH +DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Libraries + + If you develop a new library, and you want it to be of the greatest +possible use to the public, we recommend making it free software that +everyone can redistribute and change. You can do so by permitting +redistribution under these terms (or, alternatively, under the terms +of the ordinary General Public License). + + To apply these terms, attach the following notices to the library. +It is safest to attach them to the start of each source file to most +effectively convey the exclusion of warranty; and each file should +have at least the "copyright" line and a pointer to where the full +notice is found. + + + + Copyright (C) + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +Also add information on how to contact you by electronic and paper mail. + +You should also get your employer (if you work as a programmer) or +your school, if any, to sign a "copyright disclaimer" for the library, +if necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the + library `Frob' (a library for tweaking knobs) written by James + Random Hacker. + + , 1 April 1990 + Ty Coon, President of Vice + +That's all there is to it! + + diff --git a/WebCrawler/venv/Lib/site-packages/astroid-2.4.2.dist-info/INSTALLER b/WebCrawler/venv/Lib/site-packages/astroid-2.4.2.dist-info/INSTALLER new file mode 100644 index 0000000..a1b589e --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/astroid-2.4.2.dist-info/INSTALLER @@ -0,0 +1 @@ +pip diff --git a/WebCrawler/venv/Lib/site-packages/astroid-2.4.2.dist-info/METADATA b/WebCrawler/venv/Lib/site-packages/astroid-2.4.2.dist-info/METADATA new file mode 100644 index 0000000..47dad94 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/astroid-2.4.2.dist-info/METADATA @@ -0,0 +1,118 @@ +Metadata-Version: 2.1 +Name: astroid +Version: 2.4.2 +Summary: An abstract syntax tree for Python with inference support. +Home-page: https://github.com/PyCQA/astroid +Author: Python Code Quality Authority +Author-email: code-quality@python.org +License: LGPL +Platform: UNKNOWN +Classifier: Topic :: Software Development :: Libraries :: Python Modules +Classifier: Topic :: Software Development :: Quality Assurance +Classifier: Programming Language :: Python +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3.5 +Classifier: Programming Language :: Python :: 3.6 +Classifier: Programming Language :: Python :: 3.7 +Classifier: Programming Language :: Python :: 3.8 +Classifier: Programming Language :: Python :: Implementation :: CPython +Classifier: Programming Language :: Python :: Implementation :: PyPy +Requires-Python: >=3.5 +Requires-Dist: lazy-object-proxy (==1.4.*) +Requires-Dist: six (~=1.12) +Requires-Dist: wrapt (~=1.11) +Requires-Dist: typed-ast (<1.5,>=1.4.0) ; implementation_name == "cpython" and python_version < "3.8" + +Astroid +======= + +.. image:: https://travis-ci.org/PyCQA/astroid.svg?branch=master + :target: https://travis-ci.org/PyCQA/astroid + +.. image:: https://ci.appveyor.com/api/projects/status/co3u42kunguhbh6l/branch/master?svg=true + :alt: AppVeyor Build Status + :target: https://ci.appveyor.com/project/PCManticore/astroid + +.. image:: https://coveralls.io/repos/github/PyCQA/astroid/badge.svg?branch=master + :target: https://coveralls.io/github/PyCQA/astroid?branch=master + +.. image:: https://readthedocs.org/projects/astroid/badge/?version=latest + :target: http://astroid.readthedocs.io/en/latest/?badge=latest + :alt: Documentation Status + +.. image:: https://img.shields.io/badge/code%20style-black-000000.svg + :target: https://github.com/ambv/black + +.. |tideliftlogo| image:: doc/media/Tidelift_Logos_RGB_Tidelift_Shorthand_On-White_small.png + :width: 75 + :height: 60 + :alt: Tidelift + +.. list-table:: + :widths: 10 100 + + * - |tideliftlogo| + - Professional support for astroid is available as part of the `Tidelift + Subscription`_. Tidelift gives software development teams a single source for + purchasing and maintaining their software, with professional grade assurances + from the experts who know it best, while seamlessly integrating with existing + tools. + +.. _Tidelift Subscription: https://tidelift.com/subscription/pkg/pypi-astroid?utm_source=pypi-astroid&utm_medium=referral&utm_campaign=readme + + + +What's this? +------------ + +The aim of this module is to provide a common base representation of +python source code. It is currently the library powering pylint's capabilities. + +It provides a compatible representation which comes from the `_ast` +module. It rebuilds the tree generated by the builtin _ast module by +recursively walking down the AST and building an extended ast. The new +node classes have additional methods and attributes for different +usages. They include some support for static inference and local name +scopes. Furthermore, astroid can also build partial trees by inspecting living +objects. + + +Installation +------------ + +Extract the tarball, jump into the created directory and run:: + + pip install . + + +If you want to do an editable installation, you can run:: + + pip install -e . + + +If you have any questions, please mail the code-quality@python.org +mailing list for support. See +http://mail.python.org/mailman/listinfo/code-quality for subscription +information and archives. + +Documentation +------------- +http://astroid.readthedocs.io/en/latest/ + + +Python Versions +--------------- + +astroid 2.0 is currently available for Python 3 only. If you want Python 2 +support, older versions of astroid will still supported until 2020. + +Test +---- + +Tests are in the 'test' subdirectory. To launch the whole tests suite, you can use +either `tox` or `pytest`:: + + tox + pytest astroid + + diff --git a/WebCrawler/venv/Lib/site-packages/astroid-2.4.2.dist-info/RECORD b/WebCrawler/venv/Lib/site-packages/astroid-2.4.2.dist-info/RECORD new file mode 100644 index 0000000..c479925 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/astroid-2.4.2.dist-info/RECORD @@ -0,0 +1,151 @@ +astroid-2.4.2.dist-info/COPYING,sha256=qxX9UmvY3Rip5368E5ZWv00z6X_HI4zRG_YOK5uGZsY,17987 +astroid-2.4.2.dist-info/COPYING.LESSER,sha256=qb3eVhbs3R6YC0TzYGAO6Hg7H5m4zIOivrFjoKOQ6GE,26527 +astroid-2.4.2.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4 +astroid-2.4.2.dist-info/METADATA,sha256=mWlOM8vl6thD_vA7Lzn2yKmQuaX8NukwS72gEpsGM0U,3915 +astroid-2.4.2.dist-info/RECORD,, +astroid-2.4.2.dist-info/WHEEL,sha256=g4nMs7d-Xl9-xC9XovUrsDHGXt-FT0E17Yqo92DEfvY,92 +astroid-2.4.2.dist-info/top_level.txt,sha256=HsdW4O2x7ZXRj6k-agi3RaQybGLobI3VSE-jt4vQUXM,8 +astroid/__init__.py,sha256=zZF5EWBTfOtOcFd62WMbhLAtS2b_Fm-3JJW2AVUUMUI,5655 +astroid/__pkginfo__.py,sha256=G6QlkatrWucF445cQW7Kb2ZSgBzf7TiKLPwqkP0CoQU,2329 +astroid/__pycache__/__init__.cpython-39.pyc,, +astroid/__pycache__/__pkginfo__.cpython-39.pyc,, +astroid/__pycache__/_ast.cpython-39.pyc,, +astroid/__pycache__/arguments.cpython-39.pyc,, +astroid/__pycache__/as_string.cpython-39.pyc,, +astroid/__pycache__/bases.cpython-39.pyc,, +astroid/__pycache__/builder.cpython-39.pyc,, +astroid/__pycache__/context.cpython-39.pyc,, +astroid/__pycache__/decorators.cpython-39.pyc,, +astroid/__pycache__/exceptions.cpython-39.pyc,, +astroid/__pycache__/helpers.cpython-39.pyc,, +astroid/__pycache__/inference.cpython-39.pyc,, +astroid/__pycache__/manager.cpython-39.pyc,, +astroid/__pycache__/mixins.cpython-39.pyc,, +astroid/__pycache__/modutils.cpython-39.pyc,, +astroid/__pycache__/node_classes.cpython-39.pyc,, +astroid/__pycache__/nodes.cpython-39.pyc,, +astroid/__pycache__/objects.cpython-39.pyc,, +astroid/__pycache__/protocols.cpython-39.pyc,, +astroid/__pycache__/raw_building.cpython-39.pyc,, +astroid/__pycache__/rebuilder.cpython-39.pyc,, +astroid/__pycache__/scoped_nodes.cpython-39.pyc,, +astroid/__pycache__/test_utils.cpython-39.pyc,, +astroid/__pycache__/transforms.cpython-39.pyc,, +astroid/__pycache__/util.cpython-39.pyc,, +astroid/_ast.py,sha256=n1U2HblBNrMhsmrjF84HUfIgEZ3PyHyiJJOrOkgiUFk,3385 +astroid/arguments.py,sha256=KJcZn7HeEhj2fZICr-9Pi7iueCNDZRQWh0T7O-qb-AE,12558 +astroid/as_string.py,sha256=8bOZWsGG79TLmlzRzPtmHdanIAqQUFTKiYH873iMhOw,22821 +astroid/bases.py,sha256=wL9C23mHFaj7vhMqM83DdsU6kfP488aEjoFWaO8p6Cg,19175 +astroid/brain/__pycache__/brain_argparse.cpython-39.pyc,, +astroid/brain/__pycache__/brain_attrs.cpython-39.pyc,, +astroid/brain/__pycache__/brain_boto3.cpython-39.pyc,, +astroid/brain/__pycache__/brain_builtin_inference.cpython-39.pyc,, +astroid/brain/__pycache__/brain_collections.cpython-39.pyc,, +astroid/brain/__pycache__/brain_crypt.cpython-39.pyc,, +astroid/brain/__pycache__/brain_curses.cpython-39.pyc,, +astroid/brain/__pycache__/brain_dataclasses.cpython-39.pyc,, +astroid/brain/__pycache__/brain_dateutil.cpython-39.pyc,, +astroid/brain/__pycache__/brain_fstrings.cpython-39.pyc,, +astroid/brain/__pycache__/brain_functools.cpython-39.pyc,, +astroid/brain/__pycache__/brain_gi.cpython-39.pyc,, +astroid/brain/__pycache__/brain_hashlib.cpython-39.pyc,, +astroid/brain/__pycache__/brain_http.cpython-39.pyc,, +astroid/brain/__pycache__/brain_io.cpython-39.pyc,, +astroid/brain/__pycache__/brain_mechanize.cpython-39.pyc,, +astroid/brain/__pycache__/brain_multiprocessing.cpython-39.pyc,, +astroid/brain/__pycache__/brain_namedtuple_enum.cpython-39.pyc,, +astroid/brain/__pycache__/brain_nose.cpython-39.pyc,, +astroid/brain/__pycache__/brain_numpy_core_fromnumeric.cpython-39.pyc,, +astroid/brain/__pycache__/brain_numpy_core_function_base.cpython-39.pyc,, +astroid/brain/__pycache__/brain_numpy_core_multiarray.cpython-39.pyc,, +astroid/brain/__pycache__/brain_numpy_core_numeric.cpython-39.pyc,, +astroid/brain/__pycache__/brain_numpy_core_numerictypes.cpython-39.pyc,, +astroid/brain/__pycache__/brain_numpy_core_umath.cpython-39.pyc,, +astroid/brain/__pycache__/brain_numpy_ndarray.cpython-39.pyc,, +astroid/brain/__pycache__/brain_numpy_random_mtrand.cpython-39.pyc,, +astroid/brain/__pycache__/brain_numpy_utils.cpython-39.pyc,, +astroid/brain/__pycache__/brain_pkg_resources.cpython-39.pyc,, +astroid/brain/__pycache__/brain_pytest.cpython-39.pyc,, +astroid/brain/__pycache__/brain_qt.cpython-39.pyc,, +astroid/brain/__pycache__/brain_random.cpython-39.pyc,, +astroid/brain/__pycache__/brain_re.cpython-39.pyc,, +astroid/brain/__pycache__/brain_responses.cpython-39.pyc,, +astroid/brain/__pycache__/brain_scipy_signal.cpython-39.pyc,, +astroid/brain/__pycache__/brain_six.cpython-39.pyc,, +astroid/brain/__pycache__/brain_ssl.cpython-39.pyc,, +astroid/brain/__pycache__/brain_subprocess.cpython-39.pyc,, +astroid/brain/__pycache__/brain_threading.cpython-39.pyc,, +astroid/brain/__pycache__/brain_typing.cpython-39.pyc,, +astroid/brain/__pycache__/brain_uuid.cpython-39.pyc,, +astroid/brain/brain_argparse.py,sha256=5XqcThekktCIWRlWAMs-R47wkbsOUSnQlsEbLEnRpsY,1041 +astroid/brain/brain_attrs.py,sha256=k8zJqIXsIbQrncthrzyB5NtdPTktgVi9wG7nyl8xMzs,2208 +astroid/brain/brain_boto3.py,sha256=nE8Cw_S_Gp___IszRDRkhEGS6WGrKcF_gTOs3GVxH9k,862 +astroid/brain/brain_builtin_inference.py,sha256=F6_09yM2mmu_u3ad_f9Uumc-q0lDA2CY8v9-PYtUMmA,28793 +astroid/brain/brain_collections.py,sha256=Uqi4VmpLDl0ndQa9x-5tIXRtVdtI6TlwR9KNHmOqLyw,2724 +astroid/brain/brain_crypt.py,sha256=gA7Q4GVuAM4viuTGWM6SNTPQXv5Gr_mFapyKMTRcsJ0,875 +astroid/brain/brain_curses.py,sha256=tDnlCP1bEvleqCMz856yua9mM5um1p_JendFhT4rBFk,3303 +astroid/brain/brain_dataclasses.py,sha256=5WndOYSY0oi2v-Od6KdPte-FKt00LoNRH2riSB4S1os,1647 +astroid/brain/brain_dateutil.py,sha256=GwDrgbaUkKbp3ImLAvuUtPuDBeK3lPh0bNI_Wphm2_8,801 +astroid/brain/brain_fstrings.py,sha256=Jaf-G-wkLecwG4jfjjfR8MDKzgAjjn2mgrrWZQLOAd8,2126 +astroid/brain/brain_functools.py,sha256=Nljy7o2vu16S2amFot4wdTI0U76Yafq-ujVPHOdGgCE,5481 +astroid/brain/brain_gi.py,sha256=oraXhBWyCCxmPEAEvRboeTIho0--ORObvckni00_1wY,7554 +astroid/brain/brain_hashlib.py,sha256=W2cS6-rixdBcre6lPWIuSOqPLCcVji5JBlImdPbV6Qo,2292 +astroid/brain/brain_http.py,sha256=a_8eIACvZetnR2xI4ZARVMuB1WSjyyqM3rCl2DVltMo,10524 +astroid/brain/brain_io.py,sha256=wEY3vvTeV23tYxFn8HHQeyjhb7-jqzwgwNI-kl2Yu-c,1476 +astroid/brain/brain_mechanize.py,sha256=boZxoCxPGSpHC_RccO5U026hyXyhunXR55M9WM1lYTo,895 +astroid/brain/brain_multiprocessing.py,sha256=9OswtRuVBj-KemJ0yp4prz8mrkJDzOzN6n13u2MgmwM,3096 +astroid/brain/brain_namedtuple_enum.py,sha256=eZ8IaHPLLHBakywJ3q4Ogtd2Tk0gtNcAgYAloMAKTjM,15966 +astroid/brain/brain_nose.py,sha256=PSPmme611h7fC8MKuVXbiGw0HZhdmIuoxM6yieZ1OOc,2217 +astroid/brain/brain_numpy_core_fromnumeric.py,sha256=NxiHbcMyQQ3Gpx4KEA5eAmGrpjlPN5rfXjRLjHPOVkw,621 +astroid/brain/brain_numpy_core_function_base.py,sha256=7i6Kge_PviqoWXhbeRgB581xwLo0dxvO54_5UB3ppig,1168 +astroid/brain/brain_numpy_core_multiarray.py,sha256=vr-nBt3EF_z-UmHmcAlJhGorgDXPpQaYX8g4TWu_2Vw,4015 +astroid/brain/brain_numpy_core_numeric.py,sha256=M0RXOym2YC-DaZfRqGNIpr6s-Oh1KHn1okGr3PtPl0k,1384 +astroid/brain/brain_numpy_core_numerictypes.py,sha256=ltlZyQprEbVdUJpC_ERGyXyJsYwzFHx7zGYDLheIonA,8013 +astroid/brain/brain_numpy_core_umath.py,sha256=je6K3ILMen2mgkZseHCOjOgnFV6SaNgAihEi6irhbRk,4436 +astroid/brain/brain_numpy_ndarray.py,sha256=WZbRG7GgOWZ68sEr4JjZUnJHLfUc91xp2mucHeLMrnw,8463 +astroid/brain/brain_numpy_random_mtrand.py,sha256=WRy_CStllgZN2B3Dw2qxXbY4SBrCzu-cdaK230VQHaE,3278 +astroid/brain/brain_numpy_utils.py,sha256=ZYtCVmn1x6P7iwYFohdW3CCrvVT5BCNYe-7jKcz_25s,1875 +astroid/brain/brain_pkg_resources.py,sha256=S_5UED1Zg8ObEJumRdpYGnjxZzemh_G_NFj3p5NGPfc,2262 +astroid/brain/brain_pytest.py,sha256=RYp7StKnC22n4vFJkHH5h7DAFWtouIcg-ZCXQ8Oz3xk,2390 +astroid/brain/brain_qt.py,sha256=m2s4YXFrnOynWDf9omHOBLVqFriBdGJAmvarJCxiffM,2536 +astroid/brain/brain_random.py,sha256=2RZY-QEXMNWp7E6h0l0-ke-DtjKTOFlTdjiQZi3XdQc,2432 +astroid/brain/brain_re.py,sha256=le7VJHUAf80HyE_aQCh7_8FyDVK6JwNWA--c9RaMVQ8,1128 +astroid/brain/brain_responses.py,sha256=GIAGkwMEGEZ9bJHI9uBGs_UmUy3DRBtjI_j_8dLBVsc,1534 +astroid/brain/brain_scipy_signal.py,sha256=IKNnGXzEH5FKhPHM2hv57pNdo1emThWwZ0ksckdsOeE,2255 +astroid/brain/brain_six.py,sha256=79aws4au6ZQJuf-YhBT4R-9wuVOwg-u0T42aUhCswK0,6187 +astroid/brain/brain_ssl.py,sha256=9dMCTQ0JsX2gpdmBkYO7BByw3zN23oXRL8Svo2S1V28,3722 +astroid/brain/brain_subprocess.py,sha256=yp-HsE69uh7T7C1kZYciAdRMQ48aNT51lI032oDppDk,4688 +astroid/brain/brain_threading.py,sha256=dY8YwZ-zrB30_CQFYWh4AK3qUVssakgwLVwunQ6yYkU,837 +astroid/brain/brain_typing.py,sha256=iFw33beNCitCJjJNvccIY6SsFJcdKVDdl-56DxDioh0,2780 +astroid/brain/brain_uuid.py,sha256=YzfiOXavu515cS1prsKGfY4-12g_KWT0Yr4-5ti0v74,569 +astroid/builder.py,sha256=72JlHrXRF9sS5zh1mqXlRHGFg88Efc2rm9yRJKDIShA,16891 +astroid/context.py,sha256=MpwiEzWZ39a4WyqYgbSmePL2nZLqALt1gjrvi3TP35U,5169 +astroid/decorators.py,sha256=z_wTjsiMlu_ElHtYUUuxaB2F_s4G0Ks7bmtCZS3IQ_Q,4283 +astroid/exceptions.py,sha256=GLiZo9BdWILJShO-il8ra-tPZqaODMAX987F--LWv2w,6891 +astroid/helpers.py,sha256=3YoJeVLoS-66T_abDorUNHiP8m2R7-faNnpc5yPJhrc,9448 +astroid/inference.py,sha256=UiKKPRYqb5JINQIw1lSdBniHhXjCohCGAqycp2GkVI4,34686 +astroid/interpreter/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +astroid/interpreter/__pycache__/__init__.cpython-39.pyc,, +astroid/interpreter/__pycache__/dunder_lookup.cpython-39.pyc,, +astroid/interpreter/__pycache__/objectmodel.cpython-39.pyc,, +astroid/interpreter/_import/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +astroid/interpreter/_import/__pycache__/__init__.cpython-39.pyc,, +astroid/interpreter/_import/__pycache__/spec.cpython-39.pyc,, +astroid/interpreter/_import/__pycache__/util.cpython-39.pyc,, +astroid/interpreter/_import/spec.py,sha256=rZ9kX3I0xPo_pFAYWeOk2Xbi4ZIg_5bRsouZgxOXegA,11436 +astroid/interpreter/_import/util.py,sha256=inubUz6F3_kaMFaeleKUW6E6wCMIPrhU882zvwEZ02I,255 +astroid/interpreter/dunder_lookup.py,sha256=dP-AZU_aGPNt03b1ttrMglxzeU3NtgnG0MfpSLPH6sg,2155 +astroid/interpreter/objectmodel.py,sha256=bbPIaamrqrx7WHtG5YNs9dbrlDC00GrJPxPGoSTsnqg,25792 +astroid/manager.py,sha256=jmEm9uH00mPA2Y68s10xrPNbZaadv_2c-CWluB3SYoE,13701 +astroid/mixins.py,sha256=F2rv2Ow7AU3YT_2jitVJik95ZWRVK6hpf8BrkkspzUY,5571 +astroid/modutils.py,sha256=GBW5Z691eqf6VAnPsZzeQ0WYzrl-0GGTHkkZNb_9XRQ,23242 +astroid/node_classes.py,sha256=cBRkZ_u8ZoRkiXznYWq1L3I7cO-P9nGowNCy8T7Qpzk,142740 +astroid/nodes.py,sha256=WoyRe22GNVRc2TRHWOUlqdxCdOVD8GKOq9v1LpPhkr8,2978 +astroid/objects.py,sha256=caKeQPBtrrfqa1q844vkehXwpdMzvph5YJdJJdbD_m8,11035 +astroid/protocols.py,sha256=m1ZHvKvFQZSZp993na4f9s8V17kqNPRfH-XpQc8gJ7c,27396 +astroid/raw_building.py,sha256=PI70y2mPQ7JURDVyNZ6deeBFkvjNNaOQBkuo0VPbvQ4,17340 +astroid/rebuilder.py,sha256=xn82eWlxzGcDd2VACUnNNWCxArfV9HI8g67fUip0XFk,39411 +astroid/scoped_nodes.py,sha256=fUrmTyltaLCr9-i8RdC2e5dPA9O8w946Mvvft8mgXdM,98203 +astroid/test_utils.py,sha256=axGB3j6ZEaVspL1ZgPizKgWehNEREb58Z97U7mQ-GA8,2319 +astroid/transforms.py,sha256=1npwJWcQUSIjcpcWd1pc-dJhtHOyiboQHsETAIQd5co,3377 +astroid/util.py,sha256=jg5LnqbWSZTZP1KgpxGBuC6Lfwhn9Jb2T2TohXghmC0,4785 diff --git a/WebCrawler/venv/Lib/site-packages/astroid-2.4.2.dist-info/WHEEL b/WebCrawler/venv/Lib/site-packages/astroid-2.4.2.dist-info/WHEEL new file mode 100644 index 0000000..b552003 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/astroid-2.4.2.dist-info/WHEEL @@ -0,0 +1,5 @@ +Wheel-Version: 1.0 +Generator: bdist_wheel (0.34.2) +Root-Is-Purelib: true +Tag: py3-none-any + diff --git a/WebCrawler/venv/Lib/site-packages/astroid-2.4.2.dist-info/top_level.txt b/WebCrawler/venv/Lib/site-packages/astroid-2.4.2.dist-info/top_level.txt new file mode 100644 index 0000000..450d4fe --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/astroid-2.4.2.dist-info/top_level.txt @@ -0,0 +1 @@ +astroid diff --git a/WebCrawler/venv/Lib/site-packages/astroid/__init__.py b/WebCrawler/venv/Lib/site-packages/astroid/__init__.py new file mode 100644 index 0000000..e869274 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/astroid/__init__.py @@ -0,0 +1,168 @@ +# Copyright (c) 2006-2013, 2015 LOGILAB S.A. (Paris, FRANCE) +# Copyright (c) 2014 Google, Inc. +# Copyright (c) 2014 Eevee (Alex Munroe) +# Copyright (c) 2015-2016, 2018 Claudiu Popa +# Copyright (c) 2015-2016 Ceridwen +# Copyright (c) 2016 Derek Gustafson +# Copyright (c) 2016 Moises Lopez +# Copyright (c) 2018 Bryce Guinta +# Copyright (c) 2019 Nick Drozd + +# Licensed under the LGPL: https://www.gnu.org/licenses/old-licenses/lgpl-2.1.en.html +# For details: https://github.com/PyCQA/astroid/blob/master/COPYING.LESSER + +"""Python Abstract Syntax Tree New Generation + +The aim of this module is to provide a common base representation of +python source code for projects such as pychecker, pyreverse, +pylint... Well, actually the development of this library is essentially +governed by pylint's needs. + +It extends class defined in the python's _ast module with some +additional methods and attributes. Instance attributes are added by a +builder object, which can either generate extended ast (let's call +them astroid ;) by visiting an existent ast tree or by inspecting living +object. Methods are added by monkey patching ast classes. + +Main modules are: + +* nodes and scoped_nodes for more information about methods and + attributes added to different node classes + +* the manager contains a high level object to get astroid trees from + source files and living objects. It maintains a cache of previously + constructed tree for quick access + +* builder contains the class responsible to build astroid trees +""" + +import enum +import itertools +import os +import sys + +import wrapt + + +_Context = enum.Enum("Context", "Load Store Del") +Load = _Context.Load +Store = _Context.Store +Del = _Context.Del +del _Context + + +# pylint: disable=wrong-import-order,wrong-import-position +from .__pkginfo__ import version as __version__ + +# WARNING: internal imports order matters ! + +# pylint: disable=redefined-builtin + +# make all exception classes accessible from astroid package +from astroid.exceptions import * + +# make all node classes accessible from astroid package +from astroid.nodes import * + +# trigger extra monkey-patching +from astroid import inference + +# more stuff available +from astroid import raw_building +from astroid.bases import BaseInstance, Instance, BoundMethod, UnboundMethod +from astroid.node_classes import are_exclusive, unpack_infer +from astroid.scoped_nodes import builtin_lookup +from astroid.builder import parse, extract_node +from astroid.util import Uninferable + +# make a manager instance (borg) accessible from astroid package +from astroid.manager import AstroidManager + +MANAGER = AstroidManager() +del AstroidManager + +# transform utilities (filters and decorator) + + +# pylint: disable=dangerous-default-value +@wrapt.decorator +def _inference_tip_cached(func, instance, args, kwargs, _cache={}): + """Cache decorator used for inference tips""" + node = args[0] + try: + return iter(_cache[func, node]) + except KeyError: + result = func(*args, **kwargs) + # Need to keep an iterator around + original, copy = itertools.tee(result) + _cache[func, node] = list(copy) + return original + + +# pylint: enable=dangerous-default-value + + +def inference_tip(infer_function, raise_on_overwrite=False): + """Given an instance specific inference function, return a function to be + given to MANAGER.register_transform to set this inference function. + + :param bool raise_on_overwrite: Raise an `InferenceOverwriteError` + if the inference tip will overwrite another. Used for debugging + + Typical usage + + .. sourcecode:: python + + MANAGER.register_transform(Call, inference_tip(infer_named_tuple), + predicate) + + .. Note:: + + Using an inference tip will override + any previously set inference tip for the given + node. Use a predicate in the transform to prevent + excess overwrites. + """ + + def transform(node, infer_function=infer_function): + if ( + raise_on_overwrite + and node._explicit_inference is not None + and node._explicit_inference is not infer_function + ): + raise InferenceOverwriteError( + "Inference already set to {existing_inference}. " + "Trying to overwrite with {new_inference} for {node}".format( + existing_inference=infer_function, + new_inference=node._explicit_inference, + node=node, + ) + ) + # pylint: disable=no-value-for-parameter + node._explicit_inference = _inference_tip_cached(infer_function) + return node + + return transform + + +def register_module_extender(manager, module_name, get_extension_mod): + def transform(node): + extension_module = get_extension_mod() + for name, objs in extension_module.locals.items(): + node.locals[name] = objs + for obj in objs: + if obj.parent is extension_module: + obj.parent = node + + manager.register_transform(Module, transform, lambda n: n.name == module_name) + + +# load brain plugins +BRAIN_MODULES_DIR = os.path.join(os.path.dirname(__file__), "brain") +if BRAIN_MODULES_DIR not in sys.path: + # add it to the end of the list so user path take precedence + sys.path.append(BRAIN_MODULES_DIR) +# load modules in this directory +for module in os.listdir(BRAIN_MODULES_DIR): + if module.endswith(".py"): + __import__(module[:-3]) diff --git a/WebCrawler/venv/Lib/site-packages/astroid/__pkginfo__.py b/WebCrawler/venv/Lib/site-packages/astroid/__pkginfo__.py new file mode 100644 index 0000000..fd8e132 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/astroid/__pkginfo__.py @@ -0,0 +1,56 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2006-2014 LOGILAB S.A. (Paris, FRANCE) +# Copyright (c) 2014-2019 Claudiu Popa +# Copyright (c) 2014 Google, Inc. +# Copyright (c) 2015-2017 Ceridwen +# Copyright (c) 2015 Florian Bruhin +# Copyright (c) 2015 Radosław Ganczarek +# Copyright (c) 2016 Moises Lopez +# Copyright (c) 2017 Hugo +# Copyright (c) 2017 Łukasz Rogalski +# Copyright (c) 2017 Calen Pennington +# Copyright (c) 2018 Ville Skyttä +# Copyright (c) 2018 Ashley Whetter +# Copyright (c) 2018 Bryce Guinta +# Copyright (c) 2019 Uilian Ries +# Copyright (c) 2019 Thomas Hisch +# Copyright (c) 2020 Michael + +# Licensed under the LGPL: https://www.gnu.org/licenses/old-licenses/lgpl-2.1.en.html +# For details: https://github.com/PyCQA/astroid/blob/master/COPYING.LESSER + +"""astroid packaging information""" + +version = "2.4.2" +numversion = tuple(int(elem) for elem in version.split(".") if elem.isdigit()) + +extras_require = {} +install_requires = [ + "lazy_object_proxy==1.4.*", + "six~=1.12", + "wrapt~=1.11", + 'typed-ast>=1.4.0,<1.5;implementation_name== "cpython" and python_version<"3.8"', +] + +# pylint: disable=redefined-builtin; why license is a builtin anyway? +license = "LGPL" + +author = "Python Code Quality Authority" +author_email = "code-quality@python.org" +mailinglist = "mailto://%s" % author_email +web = "https://github.com/PyCQA/astroid" + +description = "An abstract syntax tree for Python with inference support." + +classifiers = [ + "Topic :: Software Development :: Libraries :: Python Modules", + "Topic :: Software Development :: Quality Assurance", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.5", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: Implementation :: CPython", + "Programming Language :: Python :: Implementation :: PyPy", +] diff --git a/WebCrawler/venv/Lib/site-packages/astroid/__pycache__/__init__.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/astroid/__pycache__/__init__.cpython-39.pyc new file mode 100644 index 0000000..3d1f13e Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/astroid/__pycache__/__init__.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/astroid/__pycache__/__pkginfo__.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/astroid/__pycache__/__pkginfo__.cpython-39.pyc new file mode 100644 index 0000000..5942fa1 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/astroid/__pycache__/__pkginfo__.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/astroid/__pycache__/_ast.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/astroid/__pycache__/_ast.cpython-39.pyc new file mode 100644 index 0000000..cd81415 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/astroid/__pycache__/_ast.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/astroid/__pycache__/arguments.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/astroid/__pycache__/arguments.cpython-39.pyc new file mode 100644 index 0000000..19c8a7f Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/astroid/__pycache__/arguments.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/astroid/__pycache__/as_string.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/astroid/__pycache__/as_string.cpython-39.pyc new file mode 100644 index 0000000..d60c7d8 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/astroid/__pycache__/as_string.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/astroid/__pycache__/bases.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/astroid/__pycache__/bases.cpython-39.pyc new file mode 100644 index 0000000..ebc08a9 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/astroid/__pycache__/bases.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/astroid/__pycache__/builder.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/astroid/__pycache__/builder.cpython-39.pyc new file mode 100644 index 0000000..47dad2f Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/astroid/__pycache__/builder.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/astroid/__pycache__/context.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/astroid/__pycache__/context.cpython-39.pyc new file mode 100644 index 0000000..c81134e Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/astroid/__pycache__/context.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/astroid/__pycache__/decorators.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/astroid/__pycache__/decorators.cpython-39.pyc new file mode 100644 index 0000000..76e14a7 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/astroid/__pycache__/decorators.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/astroid/__pycache__/exceptions.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/astroid/__pycache__/exceptions.cpython-39.pyc new file mode 100644 index 0000000..c310b4d Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/astroid/__pycache__/exceptions.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/astroid/__pycache__/helpers.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/astroid/__pycache__/helpers.cpython-39.pyc new file mode 100644 index 0000000..2dd27f1 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/astroid/__pycache__/helpers.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/astroid/__pycache__/inference.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/astroid/__pycache__/inference.cpython-39.pyc new file mode 100644 index 0000000..c7cdf33 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/astroid/__pycache__/inference.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/astroid/__pycache__/manager.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/astroid/__pycache__/manager.cpython-39.pyc new file mode 100644 index 0000000..015af69 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/astroid/__pycache__/manager.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/astroid/__pycache__/mixins.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/astroid/__pycache__/mixins.cpython-39.pyc new file mode 100644 index 0000000..5c8c27a Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/astroid/__pycache__/mixins.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/astroid/__pycache__/modutils.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/astroid/__pycache__/modutils.cpython-39.pyc new file mode 100644 index 0000000..1bc3f02 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/astroid/__pycache__/modutils.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/astroid/__pycache__/node_classes.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/astroid/__pycache__/node_classes.cpython-39.pyc new file mode 100644 index 0000000..5af979e Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/astroid/__pycache__/node_classes.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/astroid/__pycache__/nodes.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/astroid/__pycache__/nodes.cpython-39.pyc new file mode 100644 index 0000000..b326c54 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/astroid/__pycache__/nodes.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/astroid/__pycache__/objects.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/astroid/__pycache__/objects.cpython-39.pyc new file mode 100644 index 0000000..371546b Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/astroid/__pycache__/objects.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/astroid/__pycache__/protocols.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/astroid/__pycache__/protocols.cpython-39.pyc new file mode 100644 index 0000000..e7c8264 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/astroid/__pycache__/protocols.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/astroid/__pycache__/raw_building.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/astroid/__pycache__/raw_building.cpython-39.pyc new file mode 100644 index 0000000..d13d6ae Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/astroid/__pycache__/raw_building.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/astroid/__pycache__/rebuilder.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/astroid/__pycache__/rebuilder.cpython-39.pyc new file mode 100644 index 0000000..5a73620 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/astroid/__pycache__/rebuilder.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/astroid/__pycache__/scoped_nodes.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/astroid/__pycache__/scoped_nodes.cpython-39.pyc new file mode 100644 index 0000000..9318645 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/astroid/__pycache__/scoped_nodes.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/astroid/__pycache__/test_utils.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/astroid/__pycache__/test_utils.cpython-39.pyc new file mode 100644 index 0000000..8f34ea1 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/astroid/__pycache__/test_utils.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/astroid/__pycache__/transforms.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/astroid/__pycache__/transforms.cpython-39.pyc new file mode 100644 index 0000000..ce41aff Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/astroid/__pycache__/transforms.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/astroid/__pycache__/util.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/astroid/__pycache__/util.cpython-39.pyc new file mode 100644 index 0000000..502d70a Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/astroid/__pycache__/util.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/astroid/_ast.py b/WebCrawler/venv/Lib/site-packages/astroid/_ast.py new file mode 100644 index 0000000..34b74c5 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/astroid/_ast.py @@ -0,0 +1,131 @@ +import ast +from collections import namedtuple +from functools import partial +from typing import Optional +import sys + +import astroid + +_ast_py3 = None +try: + import typed_ast.ast3 as _ast_py3 +except ImportError: + pass + + +PY38 = sys.version_info[:2] >= (3, 8) +if PY38: + # On Python 3.8, typed_ast was merged back into `ast` + _ast_py3 = ast + + +FunctionType = namedtuple("FunctionType", ["argtypes", "returns"]) + + +class ParserModule( + namedtuple( + "ParserModule", + [ + "module", + "unary_op_classes", + "cmp_op_classes", + "bool_op_classes", + "bin_op_classes", + "context_classes", + ], + ) +): + def parse(self, string: str, type_comments=True): + if self.module is _ast_py3: + if PY38: + parse_func = partial(self.module.parse, type_comments=type_comments) + else: + parse_func = partial( + self.module.parse, feature_version=sys.version_info.minor + ) + else: + parse_func = self.module.parse + return parse_func(string) + + +def parse_function_type_comment(type_comment: str) -> Optional[FunctionType]: + """Given a correct type comment, obtain a FunctionType object""" + if _ast_py3 is None: + return None + + func_type = _ast_py3.parse(type_comment, "", "func_type") + return FunctionType(argtypes=func_type.argtypes, returns=func_type.returns) + + +def get_parser_module(type_comments=True) -> ParserModule: + if not type_comments: + parser_module = ast + else: + parser_module = _ast_py3 + parser_module = parser_module or ast + + unary_op_classes = _unary_operators_from_module(parser_module) + cmp_op_classes = _compare_operators_from_module(parser_module) + bool_op_classes = _bool_operators_from_module(parser_module) + bin_op_classes = _binary_operators_from_module(parser_module) + context_classes = _contexts_from_module(parser_module) + + return ParserModule( + parser_module, + unary_op_classes, + cmp_op_classes, + bool_op_classes, + bin_op_classes, + context_classes, + ) + + +def _unary_operators_from_module(module): + return {module.UAdd: "+", module.USub: "-", module.Not: "not", module.Invert: "~"} + + +def _binary_operators_from_module(module): + binary_operators = { + module.Add: "+", + module.BitAnd: "&", + module.BitOr: "|", + module.BitXor: "^", + module.Div: "/", + module.FloorDiv: "//", + module.MatMult: "@", + module.Mod: "%", + module.Mult: "*", + module.Pow: "**", + module.Sub: "-", + module.LShift: "<<", + module.RShift: ">>", + } + return binary_operators + + +def _bool_operators_from_module(module): + return {module.And: "and", module.Or: "or"} + + +def _compare_operators_from_module(module): + return { + module.Eq: "==", + module.Gt: ">", + module.GtE: ">=", + module.In: "in", + module.Is: "is", + module.IsNot: "is not", + module.Lt: "<", + module.LtE: "<=", + module.NotEq: "!=", + module.NotIn: "not in", + } + + +def _contexts_from_module(module): + return { + module.Load: astroid.Load, + module.Store: astroid.Store, + module.Del: astroid.Del, + module.Param: astroid.Store, + } diff --git a/WebCrawler/venv/Lib/site-packages/astroid/arguments.py b/WebCrawler/venv/Lib/site-packages/astroid/arguments.py new file mode 100644 index 0000000..5f4d909 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/astroid/arguments.py @@ -0,0 +1,300 @@ +# Copyright (c) 2015-2016, 2018-2020 Claudiu Popa +# Copyright (c) 2015-2016 Ceridwen +# Copyright (c) 2018 Bryce Guinta +# Copyright (c) 2018 Nick Drozd +# Copyright (c) 2018 Anthony Sottile + +# Licensed under the LGPL: https://www.gnu.org/licenses/old-licenses/lgpl-2.1.en.html +# For details: https://github.com/PyCQA/astroid/blob/master/COPYING.LESSER + + +from astroid import bases +from astroid import context as contextmod +from astroid import exceptions +from astroid import nodes +from astroid import util + + +class CallSite: + """Class for understanding arguments passed into a call site + + It needs a call context, which contains the arguments and the + keyword arguments that were passed into a given call site. + In order to infer what an argument represents, call :meth:`infer_argument` + with the corresponding function node and the argument name. + + :param callcontext: + An instance of :class:`astroid.context.CallContext`, that holds + the arguments for the call site. + :param argument_context_map: + Additional contexts per node, passed in from :attr:`astroid.context.Context.extra_context` + :param context: + An instance of :class:`astroid.context.Context`. + """ + + def __init__(self, callcontext, argument_context_map=None, context=None): + if argument_context_map is None: + argument_context_map = {} + self.argument_context_map = argument_context_map + args = callcontext.args + keywords = callcontext.keywords + self.duplicated_keywords = set() + self._unpacked_args = self._unpack_args(args, context=context) + self._unpacked_kwargs = self._unpack_keywords(keywords, context=context) + + self.positional_arguments = [ + arg for arg in self._unpacked_args if arg is not util.Uninferable + ] + self.keyword_arguments = { + key: value + for key, value in self._unpacked_kwargs.items() + if value is not util.Uninferable + } + + @classmethod + def from_call(cls, call_node, context=None): + """Get a CallSite object from the given Call node. + + :param context: + An instance of :class:`astroid.context.Context` that will be used + to force a single inference path. + """ + + # Determine the callcontext from the given `context` object if any. + context = context or contextmod.InferenceContext() + callcontext = contextmod.CallContext(call_node.args, call_node.keywords) + return cls(callcontext, context=context) + + def has_invalid_arguments(self): + """Check if in the current CallSite were passed *invalid* arguments + + This can mean multiple things. For instance, if an unpacking + of an invalid object was passed, then this method will return True. + Other cases can be when the arguments can't be inferred by astroid, + for example, by passing objects which aren't known statically. + """ + return len(self.positional_arguments) != len(self._unpacked_args) + + def has_invalid_keywords(self): + """Check if in the current CallSite were passed *invalid* keyword arguments + + For instance, unpacking a dictionary with integer keys is invalid + (**{1:2}), because the keys must be strings, which will make this + method to return True. Other cases where this might return True if + objects which can't be inferred were passed. + """ + return len(self.keyword_arguments) != len(self._unpacked_kwargs) + + def _unpack_keywords(self, keywords, context=None): + values = {} + context = context or contextmod.InferenceContext() + context.extra_context = self.argument_context_map + for name, value in keywords: + if name is None: + # Then it's an unpacking operation (**) + try: + inferred = next(value.infer(context=context)) + except exceptions.InferenceError: + values[name] = util.Uninferable + continue + + if not isinstance(inferred, nodes.Dict): + # Not something we can work with. + values[name] = util.Uninferable + continue + + for dict_key, dict_value in inferred.items: + try: + dict_key = next(dict_key.infer(context=context)) + except exceptions.InferenceError: + values[name] = util.Uninferable + continue + if not isinstance(dict_key, nodes.Const): + values[name] = util.Uninferable + continue + if not isinstance(dict_key.value, str): + values[name] = util.Uninferable + continue + if dict_key.value in values: + # The name is already in the dictionary + values[dict_key.value] = util.Uninferable + self.duplicated_keywords.add(dict_key.value) + continue + values[dict_key.value] = dict_value + else: + values[name] = value + return values + + def _unpack_args(self, args, context=None): + values = [] + context = context or contextmod.InferenceContext() + context.extra_context = self.argument_context_map + for arg in args: + if isinstance(arg, nodes.Starred): + try: + inferred = next(arg.value.infer(context=context)) + except exceptions.InferenceError: + values.append(util.Uninferable) + continue + + if inferred is util.Uninferable: + values.append(util.Uninferable) + continue + if not hasattr(inferred, "elts"): + values.append(util.Uninferable) + continue + values.extend(inferred.elts) + else: + values.append(arg) + return values + + def infer_argument(self, funcnode, name, context): + """infer a function argument value according to the call context + + Arguments: + funcnode: The function being called. + name: The name of the argument whose value is being inferred. + context: Inference context object + """ + if name in self.duplicated_keywords: + raise exceptions.InferenceError( + "The arguments passed to {func!r} " " have duplicate keywords.", + call_site=self, + func=funcnode, + arg=name, + context=context, + ) + + # Look into the keywords first, maybe it's already there. + try: + return self.keyword_arguments[name].infer(context) + except KeyError: + pass + + # Too many arguments given and no variable arguments. + if len(self.positional_arguments) > len(funcnode.args.args): + if not funcnode.args.vararg: + raise exceptions.InferenceError( + "Too many positional arguments " + "passed to {func!r} that does " + "not have *args.", + call_site=self, + func=funcnode, + arg=name, + context=context, + ) + + positional = self.positional_arguments[: len(funcnode.args.args)] + vararg = self.positional_arguments[len(funcnode.args.args) :] + argindex = funcnode.args.find_argname(name)[0] + kwonlyargs = {arg.name for arg in funcnode.args.kwonlyargs} + kwargs = { + key: value + for key, value in self.keyword_arguments.items() + if key not in kwonlyargs + } + # If there are too few positionals compared to + # what the function expects to receive, check to see + # if the missing positional arguments were passed + # as keyword arguments and if so, place them into the + # positional args list. + if len(positional) < len(funcnode.args.args): + for func_arg in funcnode.args.args: + if func_arg.name in kwargs: + arg = kwargs.pop(func_arg.name) + positional.append(arg) + + if argindex is not None: + # 2. first argument of instance/class method + if argindex == 0 and funcnode.type in ("method", "classmethod"): + if context.boundnode is not None: + boundnode = context.boundnode + else: + # XXX can do better ? + boundnode = funcnode.parent.frame() + + if isinstance(boundnode, nodes.ClassDef): + # Verify that we're accessing a method + # of the metaclass through a class, as in + # `cls.metaclass_method`. In this case, the + # first argument is always the class. + method_scope = funcnode.parent.scope() + if method_scope is boundnode.metaclass(): + return iter((boundnode,)) + + if funcnode.type == "method": + if not isinstance(boundnode, bases.Instance): + boundnode = boundnode.instantiate_class() + return iter((boundnode,)) + if funcnode.type == "classmethod": + return iter((boundnode,)) + # if we have a method, extract one position + # from the index, so we'll take in account + # the extra parameter represented by `self` or `cls` + if funcnode.type in ("method", "classmethod"): + argindex -= 1 + # 2. search arg index + try: + return self.positional_arguments[argindex].infer(context) + except IndexError: + pass + + if funcnode.args.kwarg == name: + # It wants all the keywords that were passed into + # the call site. + if self.has_invalid_keywords(): + raise exceptions.InferenceError( + "Inference failed to find values for all keyword arguments " + "to {func!r}: {unpacked_kwargs!r} doesn't correspond to " + "{keyword_arguments!r}.", + keyword_arguments=self.keyword_arguments, + unpacked_kwargs=self._unpacked_kwargs, + call_site=self, + func=funcnode, + arg=name, + context=context, + ) + kwarg = nodes.Dict( + lineno=funcnode.args.lineno, + col_offset=funcnode.args.col_offset, + parent=funcnode.args, + ) + kwarg.postinit( + [(nodes.const_factory(key), value) for key, value in kwargs.items()] + ) + return iter((kwarg,)) + if funcnode.args.vararg == name: + # It wants all the args that were passed into + # the call site. + if self.has_invalid_arguments(): + raise exceptions.InferenceError( + "Inference failed to find values for all positional " + "arguments to {func!r}: {unpacked_args!r} doesn't " + "correspond to {positional_arguments!r}.", + positional_arguments=self.positional_arguments, + unpacked_args=self._unpacked_args, + call_site=self, + func=funcnode, + arg=name, + context=context, + ) + args = nodes.Tuple( + lineno=funcnode.args.lineno, + col_offset=funcnode.args.col_offset, + parent=funcnode.args, + ) + args.postinit(vararg) + return iter((args,)) + + # Check if it's a default parameter. + try: + return funcnode.args.default_value(name).infer(context) + except exceptions.NoDefault: + pass + raise exceptions.InferenceError( + "No value found for argument {name} to " "{func!r}", + call_site=self, + func=funcnode, + arg=name, + context=context, + ) diff --git a/WebCrawler/venv/Lib/site-packages/astroid/as_string.py b/WebCrawler/venv/Lib/site-packages/astroid/as_string.py new file mode 100644 index 0000000..653411f --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/astroid/as_string.py @@ -0,0 +1,631 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2009-2011, 2013-2014 LOGILAB S.A. (Paris, FRANCE) +# Copyright (c) 2010 Daniel Harding +# Copyright (c) 2013-2016, 2018-2020 Claudiu Popa +# Copyright (c) 2013-2014 Google, Inc. +# Copyright (c) 2015-2016 Ceridwen +# Copyright (c) 2016 Jared Garst +# Copyright (c) 2016 Jakub Wilk +# Copyright (c) 2017, 2019 Łukasz Rogalski +# Copyright (c) 2017 rr- +# Copyright (c) 2018 Serhiy Storchaka +# Copyright (c) 2018 Ville Skyttä +# Copyright (c) 2018 brendanator +# Copyright (c) 2018 Nick Drozd +# Copyright (c) 2019 Alex Hall +# Copyright (c) 2019 Hugo van Kemenade + +# Licensed under the LGPL: https://www.gnu.org/licenses/old-licenses/lgpl-2.1.en.html +# For details: https://github.com/PyCQA/astroid/blob/master/COPYING.LESSER + +"""This module renders Astroid nodes as string: + +* :func:`to_code` function return equivalent (hopefully valid) python string + +* :func:`dump` function return an internal representation of nodes found + in the tree, useful for debugging or understanding the tree structure +""" + +# pylint: disable=unused-argument + +DOC_NEWLINE = "\0" + + +class AsStringVisitor: + """Visitor to render an Astroid node as a valid python code string""" + + def __init__(self, indent): + self.indent = indent + + def __call__(self, node): + """Makes this visitor behave as a simple function""" + return node.accept(self).replace(DOC_NEWLINE, "\n") + + def _docs_dedent(self, doc): + """Stop newlines in docs being indented by self._stmt_list""" + return '\n%s"""%s"""' % (self.indent, doc.replace("\n", DOC_NEWLINE)) + + def _stmt_list(self, stmts, indent=True): + """return a list of nodes to string""" + stmts = "\n".join(nstr for nstr in [n.accept(self) for n in stmts] if nstr) + if indent: + return self.indent + stmts.replace("\n", "\n" + self.indent) + + return stmts + + def _precedence_parens(self, node, child, is_left=True): + """Wrap child in parens only if required to keep same semantics""" + if self._should_wrap(node, child, is_left): + return "(%s)" % child.accept(self) + + return child.accept(self) + + def _should_wrap(self, node, child, is_left): + """Wrap child if: + - it has lower precedence + - same precedence with position opposite to associativity direction + """ + node_precedence = node.op_precedence() + child_precedence = child.op_precedence() + + if node_precedence > child_precedence: + # 3 * (4 + 5) + return True + + if ( + node_precedence == child_precedence + and is_left != node.op_left_associative() + ): + # 3 - (4 - 5) + # (2**3)**4 + return True + + return False + + ## visit_ methods ########################################### + + def visit_await(self, node): + return "await %s" % node.value.accept(self) + + def visit_asyncwith(self, node): + return "async %s" % self.visit_with(node) + + def visit_asyncfor(self, node): + return "async %s" % self.visit_for(node) + + def visit_arguments(self, node): + """return an astroid.Function node as string""" + return node.format_args() + + def visit_assignattr(self, node): + """return an astroid.AssAttr node as string""" + return self.visit_attribute(node) + + def visit_assert(self, node): + """return an astroid.Assert node as string""" + if node.fail: + return "assert %s, %s" % (node.test.accept(self), node.fail.accept(self)) + return "assert %s" % node.test.accept(self) + + def visit_assignname(self, node): + """return an astroid.AssName node as string""" + return node.name + + def visit_assign(self, node): + """return an astroid.Assign node as string""" + lhs = " = ".join(n.accept(self) for n in node.targets) + return "%s = %s" % (lhs, node.value.accept(self)) + + def visit_augassign(self, node): + """return an astroid.AugAssign node as string""" + return "%s %s %s" % (node.target.accept(self), node.op, node.value.accept(self)) + + def visit_annassign(self, node): + """Return an astroid.AugAssign node as string""" + + target = node.target.accept(self) + annotation = node.annotation.accept(self) + if node.value is None: + return "%s: %s" % (target, annotation) + return "%s: %s = %s" % (target, annotation, node.value.accept(self)) + + def visit_repr(self, node): + """return an astroid.Repr node as string""" + return "`%s`" % node.value.accept(self) + + def visit_binop(self, node): + """return an astroid.BinOp node as string""" + left = self._precedence_parens(node, node.left) + right = self._precedence_parens(node, node.right, is_left=False) + if node.op == "**": + return "%s%s%s" % (left, node.op, right) + + return "%s %s %s" % (left, node.op, right) + + def visit_boolop(self, node): + """return an astroid.BoolOp node as string""" + values = ["%s" % self._precedence_parens(node, n) for n in node.values] + return (" %s " % node.op).join(values) + + def visit_break(self, node): + """return an astroid.Break node as string""" + return "break" + + def visit_call(self, node): + """return an astroid.Call node as string""" + expr_str = self._precedence_parens(node, node.func) + args = [arg.accept(self) for arg in node.args] + if node.keywords: + keywords = [kwarg.accept(self) for kwarg in node.keywords] + else: + keywords = [] + + args.extend(keywords) + return "%s(%s)" % (expr_str, ", ".join(args)) + + def visit_classdef(self, node): + """return an astroid.ClassDef node as string""" + decorate = node.decorators.accept(self) if node.decorators else "" + args = [n.accept(self) for n in node.bases] + if node._metaclass and not node.has_metaclass_hack(): + args.append("metaclass=" + node._metaclass.accept(self)) + args += [n.accept(self) for n in node.keywords] + args = "(%s)" % ", ".join(args) if args else "" + docs = self._docs_dedent(node.doc) if node.doc else "" + return "\n\n%sclass %s%s:%s\n%s\n" % ( + decorate, + node.name, + args, + docs, + self._stmt_list(node.body), + ) + + def visit_compare(self, node): + """return an astroid.Compare node as string""" + rhs_str = " ".join( + [ + "%s %s" % (op, self._precedence_parens(node, expr, is_left=False)) + for op, expr in node.ops + ] + ) + return "%s %s" % (self._precedence_parens(node, node.left), rhs_str) + + def visit_comprehension(self, node): + """return an astroid.Comprehension node as string""" + ifs = "".join(" if %s" % n.accept(self) for n in node.ifs) + generated = "for %s in %s%s" % ( + node.target.accept(self), + node.iter.accept(self), + ifs, + ) + return "%s%s" % ("async " if node.is_async else "", generated) + + def visit_const(self, node): + """return an astroid.Const node as string""" + if node.value is Ellipsis: + return "..." + return repr(node.value) + + def visit_continue(self, node): + """return an astroid.Continue node as string""" + return "continue" + + def visit_delete(self, node): # XXX check if correct + """return an astroid.Delete node as string""" + return "del %s" % ", ".join(child.accept(self) for child in node.targets) + + def visit_delattr(self, node): + """return an astroid.DelAttr node as string""" + return self.visit_attribute(node) + + def visit_delname(self, node): + """return an astroid.DelName node as string""" + return node.name + + def visit_decorators(self, node): + """return an astroid.Decorators node as string""" + return "@%s\n" % "\n@".join(item.accept(self) for item in node.nodes) + + def visit_dict(self, node): + """return an astroid.Dict node as string""" + return "{%s}" % ", ".join(self._visit_dict(node)) + + def _visit_dict(self, node): + for key, value in node.items: + key = key.accept(self) + value = value.accept(self) + if key == "**": + # It can only be a DictUnpack node. + yield key + value + else: + yield "%s: %s" % (key, value) + + def visit_dictunpack(self, node): + return "**" + + def visit_dictcomp(self, node): + """return an astroid.DictComp node as string""" + return "{%s: %s %s}" % ( + node.key.accept(self), + node.value.accept(self), + " ".join(n.accept(self) for n in node.generators), + ) + + def visit_expr(self, node): + """return an astroid.Discard node as string""" + return node.value.accept(self) + + def visit_emptynode(self, node): + """dummy method for visiting an Empty node""" + return "" + + def visit_excepthandler(self, node): + if node.type: + if node.name: + excs = "except %s as %s" % ( + node.type.accept(self), + node.name.accept(self), + ) + else: + excs = "except %s" % node.type.accept(self) + else: + excs = "except" + return "%s:\n%s" % (excs, self._stmt_list(node.body)) + + def visit_ellipsis(self, node): + """return an astroid.Ellipsis node as string""" + return "..." + + def visit_empty(self, node): + """return an Empty node as string""" + return "" + + def visit_exec(self, node): + """return an astroid.Exec node as string""" + if node.locals: + return "exec %s in %s, %s" % ( + node.expr.accept(self), + node.locals.accept(self), + node.globals.accept(self), + ) + if node.globals: + return "exec %s in %s" % (node.expr.accept(self), node.globals.accept(self)) + return "exec %s" % node.expr.accept(self) + + def visit_extslice(self, node): + """return an astroid.ExtSlice node as string""" + return ", ".join(dim.accept(self) for dim in node.dims) + + def visit_for(self, node): + """return an astroid.For node as string""" + fors = "for %s in %s:\n%s" % ( + node.target.accept(self), + node.iter.accept(self), + self._stmt_list(node.body), + ) + if node.orelse: + fors = "%s\nelse:\n%s" % (fors, self._stmt_list(node.orelse)) + return fors + + def visit_importfrom(self, node): + """return an astroid.ImportFrom node as string""" + return "from %s import %s" % ( + "." * (node.level or 0) + node.modname, + _import_string(node.names), + ) + + def visit_joinedstr(self, node): + string = "".join( + # Use repr on the string literal parts + # to get proper escapes, e.g. \n, \\, \" + # But strip the quotes off the ends + # (they will always be one character: ' or ") + repr(value.value)[1:-1] + # Literal braces must be doubled to escape them + .replace("{", "{{").replace("}", "}}") + # Each value in values is either a string literal (Const) + # or a FormattedValue + if type(value).__name__ == "Const" else value.accept(self) + for value in node.values + ) + + # Try to find surrounding quotes that don't appear at all in the string. + # Because the formatted values inside {} can't contain backslash (\) + # using a triple quote is sometimes necessary + for quote in ["'", '"', '"""', "'''"]: + if quote not in string: + break + + return "f" + quote + string + quote + + def visit_formattedvalue(self, node): + result = node.value.accept(self) + if node.conversion and node.conversion >= 0: + # e.g. if node.conversion == 114: result += "!r" + result += "!" + chr(node.conversion) + if node.format_spec: + # The format spec is itself a JoinedString, i.e. an f-string + # We strip the f and quotes of the ends + result += ":" + node.format_spec.accept(self)[2:-1] + return "{%s}" % result + + def handle_functiondef(self, node, keyword): + """return a (possibly async) function definition node as string""" + decorate = node.decorators.accept(self) if node.decorators else "" + docs = self._docs_dedent(node.doc) if node.doc else "" + trailer = ":" + if node.returns: + return_annotation = " -> " + node.returns.as_string() + trailer = return_annotation + ":" + def_format = "\n%s%s %s(%s)%s%s\n%s" + return def_format % ( + decorate, + keyword, + node.name, + node.args.accept(self), + trailer, + docs, + self._stmt_list(node.body), + ) + + def visit_functiondef(self, node): + """return an astroid.FunctionDef node as string""" + return self.handle_functiondef(node, "def") + + def visit_asyncfunctiondef(self, node): + """return an astroid.AsyncFunction node as string""" + return self.handle_functiondef(node, "async def") + + def visit_generatorexp(self, node): + """return an astroid.GeneratorExp node as string""" + return "(%s %s)" % ( + node.elt.accept(self), + " ".join(n.accept(self) for n in node.generators), + ) + + def visit_attribute(self, node): + """return an astroid.Getattr node as string""" + left = self._precedence_parens(node, node.expr) + if left.isdigit(): + left = "(%s)" % left + return "%s.%s" % (left, node.attrname) + + def visit_global(self, node): + """return an astroid.Global node as string""" + return "global %s" % ", ".join(node.names) + + def visit_if(self, node): + """return an astroid.If node as string""" + ifs = ["if %s:\n%s" % (node.test.accept(self), self._stmt_list(node.body))] + if node.has_elif_block(): + ifs.append("el%s" % self._stmt_list(node.orelse, indent=False)) + elif node.orelse: + ifs.append("else:\n%s" % self._stmt_list(node.orelse)) + return "\n".join(ifs) + + def visit_ifexp(self, node): + """return an astroid.IfExp node as string""" + return "%s if %s else %s" % ( + self._precedence_parens(node, node.body, is_left=True), + self._precedence_parens(node, node.test, is_left=True), + self._precedence_parens(node, node.orelse, is_left=False), + ) + + def visit_import(self, node): + """return an astroid.Import node as string""" + return "import %s" % _import_string(node.names) + + def visit_keyword(self, node): + """return an astroid.Keyword node as string""" + if node.arg is None: + return "**%s" % node.value.accept(self) + return "%s=%s" % (node.arg, node.value.accept(self)) + + def visit_lambda(self, node): + """return an astroid.Lambda node as string""" + args = node.args.accept(self) + body = node.body.accept(self) + if args: + return "lambda %s: %s" % (args, body) + + return "lambda: %s" % body + + def visit_list(self, node): + """return an astroid.List node as string""" + return "[%s]" % ", ".join(child.accept(self) for child in node.elts) + + def visit_listcomp(self, node): + """return an astroid.ListComp node as string""" + return "[%s %s]" % ( + node.elt.accept(self), + " ".join(n.accept(self) for n in node.generators), + ) + + def visit_module(self, node): + """return an astroid.Module node as string""" + docs = '"""%s"""\n\n' % node.doc if node.doc else "" + return docs + "\n".join(n.accept(self) for n in node.body) + "\n\n" + + def visit_name(self, node): + """return an astroid.Name node as string""" + return node.name + + def visit_namedexpr(self, node): + """Return an assignment expression node as string""" + target = node.target.accept(self) + value = node.value.accept(self) + return "%s := %s" % (target, value) + + def visit_nonlocal(self, node): + """return an astroid.Nonlocal node as string""" + return "nonlocal %s" % ", ".join(node.names) + + def visit_pass(self, node): + """return an astroid.Pass node as string""" + return "pass" + + def visit_print(self, node): + """return an astroid.Print node as string""" + nodes = ", ".join(n.accept(self) for n in node.values) + if not node.nl: + nodes = "%s," % nodes + if node.dest: + return "print >> %s, %s" % (node.dest.accept(self), nodes) + return "print %s" % nodes + + def visit_raise(self, node): + """return an astroid.Raise node as string""" + if node.exc: + if node.cause: + return "raise %s from %s" % ( + node.exc.accept(self), + node.cause.accept(self), + ) + return "raise %s" % node.exc.accept(self) + return "raise" + + def visit_return(self, node): + """return an astroid.Return node as string""" + if node.is_tuple_return() and len(node.value.elts) > 1: + elts = [child.accept(self) for child in node.value.elts] + return "return %s" % ", ".join(elts) + + if node.value: + return "return %s" % node.value.accept(self) + + return "return" + + def visit_index(self, node): + """return an astroid.Index node as string""" + return node.value.accept(self) + + def visit_set(self, node): + """return an astroid.Set node as string""" + return "{%s}" % ", ".join(child.accept(self) for child in node.elts) + + def visit_setcomp(self, node): + """return an astroid.SetComp node as string""" + return "{%s %s}" % ( + node.elt.accept(self), + " ".join(n.accept(self) for n in node.generators), + ) + + def visit_slice(self, node): + """return an astroid.Slice node as string""" + lower = node.lower.accept(self) if node.lower else "" + upper = node.upper.accept(self) if node.upper else "" + step = node.step.accept(self) if node.step else "" + if step: + return "%s:%s:%s" % (lower, upper, step) + return "%s:%s" % (lower, upper) + + def visit_subscript(self, node): + """return an astroid.Subscript node as string""" + idx = node.slice + if idx.__class__.__name__.lower() == "index": + idx = idx.value + idxstr = idx.accept(self) + if idx.__class__.__name__.lower() == "tuple" and idx.elts: + # Remove parenthesis in tuple and extended slice. + # a[(::1, 1:)] is not valid syntax. + idxstr = idxstr[1:-1] + return "%s[%s]" % (self._precedence_parens(node, node.value), idxstr) + + def visit_tryexcept(self, node): + """return an astroid.TryExcept node as string""" + trys = ["try:\n%s" % self._stmt_list(node.body)] + for handler in node.handlers: + trys.append(handler.accept(self)) + if node.orelse: + trys.append("else:\n%s" % self._stmt_list(node.orelse)) + return "\n".join(trys) + + def visit_tryfinally(self, node): + """return an astroid.TryFinally node as string""" + return "try:\n%s\nfinally:\n%s" % ( + self._stmt_list(node.body), + self._stmt_list(node.finalbody), + ) + + def visit_tuple(self, node): + """return an astroid.Tuple node as string""" + if len(node.elts) == 1: + return "(%s, )" % node.elts[0].accept(self) + return "(%s)" % ", ".join(child.accept(self) for child in node.elts) + + def visit_unaryop(self, node): + """return an astroid.UnaryOp node as string""" + if node.op == "not": + operator = "not " + else: + operator = node.op + return "%s%s" % (operator, self._precedence_parens(node, node.operand)) + + def visit_while(self, node): + """return an astroid.While node as string""" + whiles = "while %s:\n%s" % (node.test.accept(self), self._stmt_list(node.body)) + if node.orelse: + whiles = "%s\nelse:\n%s" % (whiles, self._stmt_list(node.orelse)) + return whiles + + def visit_with(self, node): # 'with' without 'as' is possible + """return an astroid.With node as string""" + items = ", ".join( + ("%s" % expr.accept(self)) + (vars and " as %s" % (vars.accept(self)) or "") + for expr, vars in node.items + ) + return "with %s:\n%s" % (items, self._stmt_list(node.body)) + + def visit_yield(self, node): + """yield an ast.Yield node as string""" + yi_val = (" " + node.value.accept(self)) if node.value else "" + expr = "yield" + yi_val + if node.parent.is_statement: + return expr + + return "(%s)" % (expr,) + + def visit_yieldfrom(self, node): + """ Return an astroid.YieldFrom node as string. """ + yi_val = (" " + node.value.accept(self)) if node.value else "" + expr = "yield from" + yi_val + if node.parent.is_statement: + return expr + + return "(%s)" % (expr,) + + def visit_starred(self, node): + """return Starred node as string""" + return "*" + node.value.accept(self) + + # These aren't for real AST nodes, but for inference objects. + + def visit_frozenset(self, node): + return node.parent.accept(self) + + def visit_super(self, node): + return node.parent.accept(self) + + def visit_uninferable(self, node): + return str(node) + + def visit_property(self, node): + return node.function.accept(self) + + def visit_evaluatedobject(self, node): + return node.original.accept(self) + + +def _import_string(names): + """return a list of (name, asname) formatted as a string""" + _names = [] + for name, asname in names: + if asname is not None: + _names.append("%s as %s" % (name, asname)) + else: + _names.append(name) + return ", ".join(_names) + + +# This sets the default indent to 4 spaces. +to_code = AsStringVisitor(" ") diff --git a/WebCrawler/venv/Lib/site-packages/astroid/bases.py b/WebCrawler/venv/Lib/site-packages/astroid/bases.py new file mode 100644 index 0000000..9c74303 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/astroid/bases.py @@ -0,0 +1,548 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2009-2011, 2013-2014 LOGILAB S.A. (Paris, FRANCE) +# Copyright (c) 2012 FELD Boris +# Copyright (c) 2014-2020 Claudiu Popa +# Copyright (c) 2014 Google, Inc. +# Copyright (c) 2014 Eevee (Alex Munroe) +# Copyright (c) 2015-2016 Ceridwen +# Copyright (c) 2015 Florian Bruhin +# Copyright (c) 2016-2017 Derek Gustafson +# Copyright (c) 2017 Calen Pennington +# Copyright (c) 2018-2019 hippo91 +# Copyright (c) 2018 Ville Skyttä +# Copyright (c) 2018 Bryce Guinta +# Copyright (c) 2018 Nick Drozd +# Copyright (c) 2018 Daniel Colascione +# Copyright (c) 2019 Hugo van Kemenade + +# Licensed under the LGPL: https://www.gnu.org/licenses/old-licenses/lgpl-2.1.en.html +# For details: https://github.com/PyCQA/astroid/blob/master/COPYING.LESSER + +"""This module contains base classes and functions for the nodes and some +inference utils. +""" + +import builtins +import collections + +from astroid import context as contextmod +from astroid import exceptions +from astroid import util + +objectmodel = util.lazy_import("interpreter.objectmodel") +helpers = util.lazy_import("helpers") +BUILTINS = builtins.__name__ +manager = util.lazy_import("manager") +MANAGER = manager.AstroidManager() + +# TODO: check if needs special treatment +BUILTINS = "builtins" +BOOL_SPECIAL_METHOD = "__bool__" + +PROPERTIES = {BUILTINS + ".property", "abc.abstractproperty"} +# List of possible property names. We use this list in order +# to see if a method is a property or not. This should be +# pretty reliable and fast, the alternative being to check each +# decorator to see if its a real property-like descriptor, which +# can be too complicated. +# Also, these aren't qualified, because each project can +# define them, we shouldn't expect to know every possible +# property-like decorator! +POSSIBLE_PROPERTIES = { + "cached_property", + "cachedproperty", + "lazyproperty", + "lazy_property", + "reify", + "lazyattribute", + "lazy_attribute", + "LazyProperty", + "lazy", + "cache_readonly", +} + + +def _is_property(meth, context=None): + decoratornames = meth.decoratornames(context=context) + if PROPERTIES.intersection(decoratornames): + return True + stripped = { + name.split(".")[-1] for name in decoratornames if name is not util.Uninferable + } + if any(name in stripped for name in POSSIBLE_PROPERTIES): + return True + + # Lookup for subclasses of *property* + if not meth.decorators: + return False + for decorator in meth.decorators.nodes or (): + inferred = helpers.safe_infer(decorator, context=context) + if inferred is None or inferred is util.Uninferable: + continue + if inferred.__class__.__name__ == "ClassDef": + for base_class in inferred.bases: + if base_class.__class__.__name__ != "Name": + continue + module, _ = base_class.lookup(base_class.name) + if module.name == BUILTINS and base_class.name == "property": + return True + + return False + + +class Proxy: + """a simple proxy object + + Note: + + Subclasses of this object will need a custom __getattr__ + if new instance attributes are created. See the Const class + """ + + _proxied = None # proxied object may be set by class or by instance + + def __init__(self, proxied=None): + if proxied is not None: + self._proxied = proxied + + def __getattr__(self, name): + if name == "_proxied": + return getattr(self.__class__, "_proxied") + if name in self.__dict__: + return self.__dict__[name] + return getattr(self._proxied, name) + + def infer(self, context=None): + yield self + + +def _infer_stmts(stmts, context, frame=None): + """Return an iterator on statements inferred by each statement in *stmts*.""" + inferred = False + if context is not None: + name = context.lookupname + context = context.clone() + else: + name = None + context = contextmod.InferenceContext() + + for stmt in stmts: + if stmt is util.Uninferable: + yield stmt + inferred = True + continue + context.lookupname = stmt._infer_name(frame, name) + try: + for inferred in stmt.infer(context=context): + yield inferred + inferred = True + except exceptions.NameInferenceError: + continue + except exceptions.InferenceError: + yield util.Uninferable + inferred = True + if not inferred: + raise exceptions.InferenceError( + "Inference failed for all members of {stmts!r}.", + stmts=stmts, + frame=frame, + context=context, + ) + + +def _infer_method_result_truth(instance, method_name, context): + # Get the method from the instance and try to infer + # its return's truth value. + meth = next(instance.igetattr(method_name, context=context), None) + if meth and hasattr(meth, "infer_call_result"): + if not meth.callable(): + return util.Uninferable + try: + for value in meth.infer_call_result(instance, context=context): + if value is util.Uninferable: + return value + + inferred = next(value.infer(context=context)) + return inferred.bool_value() + except exceptions.InferenceError: + pass + return util.Uninferable + + +class BaseInstance(Proxy): + """An instance base class, which provides lookup methods for potential instances.""" + + special_attributes = None + + def display_type(self): + return "Instance of" + + def getattr(self, name, context=None, lookupclass=True): + try: + values = self._proxied.instance_attr(name, context) + except exceptions.AttributeInferenceError as exc: + if self.special_attributes and name in self.special_attributes: + return [self.special_attributes.lookup(name)] + + if lookupclass: + # Class attributes not available through the instance + # unless they are explicitly defined. + return self._proxied.getattr(name, context, class_context=False) + + raise exceptions.AttributeInferenceError( + target=self, attribute=name, context=context + ) from exc + # since we've no context information, return matching class members as + # well + if lookupclass: + try: + return values + self._proxied.getattr( + name, context, class_context=False + ) + except exceptions.AttributeInferenceError: + pass + return values + + def igetattr(self, name, context=None): + """inferred getattr""" + if not context: + context = contextmod.InferenceContext() + try: + # avoid recursively inferring the same attr on the same class + if context.push((self._proxied, name)): + raise exceptions.InferenceError( + message="Cannot infer the same attribute again", + node=self, + context=context, + ) + + # XXX frame should be self._proxied, or not ? + get_attr = self.getattr(name, context, lookupclass=False) + yield from _infer_stmts( + self._wrap_attr(get_attr, context), context, frame=self + ) + except exceptions.AttributeInferenceError as error: + try: + # fallback to class.igetattr since it has some logic to handle + # descriptors + # But only if the _proxied is the Class. + if self._proxied.__class__.__name__ != "ClassDef": + raise + attrs = self._proxied.igetattr(name, context, class_context=False) + yield from self._wrap_attr(attrs, context) + except exceptions.AttributeInferenceError as error: + raise exceptions.InferenceError(**vars(error)) from error + + def _wrap_attr(self, attrs, context=None): + """wrap bound methods of attrs in a InstanceMethod proxies""" + for attr in attrs: + if isinstance(attr, UnboundMethod): + if _is_property(attr): + yield from attr.infer_call_result(self, context) + else: + yield BoundMethod(attr, self) + elif hasattr(attr, "name") and attr.name == "": + if attr.args.arguments and attr.args.arguments[0].name == "self": + yield BoundMethod(attr, self) + continue + yield attr + else: + yield attr + + def infer_call_result(self, caller, context=None): + """infer what a class instance is returning when called""" + context = contextmod.bind_context_to_node(context, self) + inferred = False + for node in self._proxied.igetattr("__call__", context): + if node is util.Uninferable or not node.callable(): + continue + for res in node.infer_call_result(caller, context): + inferred = True + yield res + if not inferred: + raise exceptions.InferenceError(node=self, caller=caller, context=context) + + +class Instance(BaseInstance): + """A special node representing a class instance.""" + + # pylint: disable=unnecessary-lambda + special_attributes = util.lazy_descriptor(lambda: objectmodel.InstanceModel()) + + def __repr__(self): + return "" % ( + self._proxied.root().name, + self._proxied.name, + id(self), + ) + + def __str__(self): + return "Instance of %s.%s" % (self._proxied.root().name, self._proxied.name) + + def callable(self): + try: + self._proxied.getattr("__call__", class_context=False) + return True + except exceptions.AttributeInferenceError: + return False + + def pytype(self): + return self._proxied.qname() + + def display_type(self): + return "Instance of" + + def bool_value(self, context=None): + """Infer the truth value for an Instance + + The truth value of an instance is determined by these conditions: + + * if it implements __bool__ on Python 3 or __nonzero__ + on Python 2, then its bool value will be determined by + calling this special method and checking its result. + * when this method is not defined, __len__() is called, if it + is defined, and the object is considered true if its result is + nonzero. If a class defines neither __len__() nor __bool__(), + all its instances are considered true. + """ + context = context or contextmod.InferenceContext() + context.callcontext = contextmod.CallContext(args=[]) + context.boundnode = self + + try: + result = _infer_method_result_truth(self, BOOL_SPECIAL_METHOD, context) + except (exceptions.InferenceError, exceptions.AttributeInferenceError): + # Fallback to __len__. + try: + result = _infer_method_result_truth(self, "__len__", context) + except (exceptions.AttributeInferenceError, exceptions.InferenceError): + return True + return result + + # This is set in inference.py. + def getitem(self, index, context=None): + pass + + +class UnboundMethod(Proxy): + """a special node representing a method not bound to an instance""" + + # pylint: disable=unnecessary-lambda + special_attributes = util.lazy_descriptor(lambda: objectmodel.UnboundMethodModel()) + + def __repr__(self): + frame = self._proxied.parent.frame() + return "<%s %s of %s at 0x%s" % ( + self.__class__.__name__, + self._proxied.name, + frame.qname(), + id(self), + ) + + def implicit_parameters(self): + return 0 + + def is_bound(self): + return False + + def getattr(self, name, context=None): + if name in self.special_attributes: + return [self.special_attributes.lookup(name)] + return self._proxied.getattr(name, context) + + def igetattr(self, name, context=None): + if name in self.special_attributes: + return iter((self.special_attributes.lookup(name),)) + return self._proxied.igetattr(name, context) + + def infer_call_result(self, caller, context): + """ + The boundnode of the regular context with a function called + on ``object.__new__`` will be of type ``object``, + which is incorrect for the argument in general. + If no context is given the ``object.__new__`` call argument will + correctly inferred except when inside a call that requires + the additional context (such as a classmethod) of the boundnode + to determine which class the method was called from + """ + + # If we're unbound method __new__ of builtin object, the result is an + # instance of the class given as first argument. + if ( + self._proxied.name == "__new__" + and self._proxied.parent.frame().qname() == "%s.object" % BUILTINS + ): + if caller.args: + node_context = context.extra_context.get(caller.args[0]) + infer = caller.args[0].infer(context=node_context) + else: + infer = [] + return (Instance(x) if x is not util.Uninferable else x for x in infer) + return self._proxied.infer_call_result(caller, context) + + def bool_value(self, context=None): + return True + + +class BoundMethod(UnboundMethod): + """a special node representing a method bound to an instance""" + + # pylint: disable=unnecessary-lambda + special_attributes = util.lazy_descriptor(lambda: objectmodel.BoundMethodModel()) + + def __init__(self, proxy, bound): + UnboundMethod.__init__(self, proxy) + self.bound = bound + + def implicit_parameters(self): + if self.name == "__new__": + # __new__ acts as a classmethod but the class argument is not implicit. + return 0 + return 1 + + def is_bound(self): + return True + + def _infer_type_new_call(self, caller, context): + """Try to infer what type.__new__(mcs, name, bases, attrs) returns. + + In order for such call to be valid, the metaclass needs to be + a subtype of ``type``, the name needs to be a string, the bases + needs to be a tuple of classes + """ + # pylint: disable=import-outside-toplevel; circular import + from astroid import node_classes + + # Verify the metaclass + mcs = next(caller.args[0].infer(context=context)) + if mcs.__class__.__name__ != "ClassDef": + # Not a valid first argument. + return None + if not mcs.is_subtype_of("%s.type" % BUILTINS): + # Not a valid metaclass. + return None + + # Verify the name + name = next(caller.args[1].infer(context=context)) + if name.__class__.__name__ != "Const": + # Not a valid name, needs to be a const. + return None + if not isinstance(name.value, str): + # Needs to be a string. + return None + + # Verify the bases + bases = next(caller.args[2].infer(context=context)) + if bases.__class__.__name__ != "Tuple": + # Needs to be a tuple. + return None + inferred_bases = [next(elt.infer(context=context)) for elt in bases.elts] + if any(base.__class__.__name__ != "ClassDef" for base in inferred_bases): + # All the bases needs to be Classes + return None + + # Verify the attributes. + attrs = next(caller.args[3].infer(context=context)) + if attrs.__class__.__name__ != "Dict": + # Needs to be a dictionary. + return None + cls_locals = collections.defaultdict(list) + for key, value in attrs.items: + key = next(key.infer(context=context)) + value = next(value.infer(context=context)) + # Ignore non string keys + if key.__class__.__name__ == "Const" and isinstance(key.value, str): + cls_locals[key.value].append(value) + + # Build the class from now. + cls = mcs.__class__( + name=name.value, + lineno=caller.lineno, + col_offset=caller.col_offset, + parent=caller, + ) + empty = node_classes.Pass() + cls.postinit( + bases=bases.elts, + body=[empty], + decorators=[], + newstyle=True, + metaclass=mcs, + keywords=[], + ) + cls.locals = cls_locals + return cls + + def infer_call_result(self, caller, context=None): + context = contextmod.bind_context_to_node(context, self.bound) + if ( + self.bound.__class__.__name__ == "ClassDef" + and self.bound.name == "type" + and self.name == "__new__" + and len(caller.args) == 4 + ): + # Check if we have a ``type.__new__(mcs, name, bases, attrs)`` call. + new_cls = self._infer_type_new_call(caller, context) + if new_cls: + return iter((new_cls,)) + + return super().infer_call_result(caller, context) + + def bool_value(self, context=None): + return True + + +class Generator(BaseInstance): + """a special node representing a generator. + + Proxied class is set once for all in raw_building. + """ + + # pylint: disable=unnecessary-lambda + special_attributes = util.lazy_descriptor(lambda: objectmodel.GeneratorModel()) + + # pylint: disable=super-init-not-called + def __init__(self, parent=None): + self.parent = parent + + def callable(self): + return False + + def pytype(self): + return "%s.generator" % BUILTINS + + def display_type(self): + return "Generator" + + def bool_value(self, context=None): + return True + + def __repr__(self): + return "" % ( + self._proxied.name, + self.lineno, + id(self), + ) + + def __str__(self): + return "Generator(%s)" % (self._proxied.name) + + +class AsyncGenerator(Generator): + """Special node representing an async generator""" + + def pytype(self): + return "%s.async_generator" % BUILTINS + + def display_type(self): + return "AsyncGenerator" + + def __repr__(self): + return "" % ( + self._proxied.name, + self.lineno, + id(self), + ) + + def __str__(self): + return "AsyncGenerator(%s)" % (self._proxied.name) diff --git a/WebCrawler/venv/Lib/site-packages/astroid/brain/__pycache__/brain_argparse.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/astroid/brain/__pycache__/brain_argparse.cpython-39.pyc new file mode 100644 index 0000000..0fead15 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/astroid/brain/__pycache__/brain_argparse.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/astroid/brain/__pycache__/brain_attrs.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/astroid/brain/__pycache__/brain_attrs.cpython-39.pyc new file mode 100644 index 0000000..0b72d68 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/astroid/brain/__pycache__/brain_attrs.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/astroid/brain/__pycache__/brain_boto3.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/astroid/brain/__pycache__/brain_boto3.cpython-39.pyc new file mode 100644 index 0000000..cdcbd90 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/astroid/brain/__pycache__/brain_boto3.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/astroid/brain/__pycache__/brain_builtin_inference.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/astroid/brain/__pycache__/brain_builtin_inference.cpython-39.pyc new file mode 100644 index 0000000..4a63f66 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/astroid/brain/__pycache__/brain_builtin_inference.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/astroid/brain/__pycache__/brain_collections.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/astroid/brain/__pycache__/brain_collections.cpython-39.pyc new file mode 100644 index 0000000..c8af0eb Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/astroid/brain/__pycache__/brain_collections.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/astroid/brain/__pycache__/brain_crypt.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/astroid/brain/__pycache__/brain_crypt.cpython-39.pyc new file mode 100644 index 0000000..8e2097b Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/astroid/brain/__pycache__/brain_crypt.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/astroid/brain/__pycache__/brain_curses.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/astroid/brain/__pycache__/brain_curses.cpython-39.pyc new file mode 100644 index 0000000..903483e Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/astroid/brain/__pycache__/brain_curses.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/astroid/brain/__pycache__/brain_dataclasses.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/astroid/brain/__pycache__/brain_dataclasses.cpython-39.pyc new file mode 100644 index 0000000..0ccc8a1 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/astroid/brain/__pycache__/brain_dataclasses.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/astroid/brain/__pycache__/brain_dateutil.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/astroid/brain/__pycache__/brain_dateutil.cpython-39.pyc new file mode 100644 index 0000000..4dc121f Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/astroid/brain/__pycache__/brain_dateutil.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/astroid/brain/__pycache__/brain_fstrings.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/astroid/brain/__pycache__/brain_fstrings.cpython-39.pyc new file mode 100644 index 0000000..bae3ccc Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/astroid/brain/__pycache__/brain_fstrings.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/astroid/brain/__pycache__/brain_functools.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/astroid/brain/__pycache__/brain_functools.cpython-39.pyc new file mode 100644 index 0000000..d061453 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/astroid/brain/__pycache__/brain_functools.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/astroid/brain/__pycache__/brain_gi.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/astroid/brain/__pycache__/brain_gi.cpython-39.pyc new file mode 100644 index 0000000..e018a64 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/astroid/brain/__pycache__/brain_gi.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/astroid/brain/__pycache__/brain_hashlib.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/astroid/brain/__pycache__/brain_hashlib.cpython-39.pyc new file mode 100644 index 0000000..851a618 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/astroid/brain/__pycache__/brain_hashlib.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/astroid/brain/__pycache__/brain_http.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/astroid/brain/__pycache__/brain_http.cpython-39.pyc new file mode 100644 index 0000000..29812d4 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/astroid/brain/__pycache__/brain_http.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/astroid/brain/__pycache__/brain_io.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/astroid/brain/__pycache__/brain_io.cpython-39.pyc new file mode 100644 index 0000000..14311d5 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/astroid/brain/__pycache__/brain_io.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/astroid/brain/__pycache__/brain_mechanize.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/astroid/brain/__pycache__/brain_mechanize.cpython-39.pyc new file mode 100644 index 0000000..abebba3 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/astroid/brain/__pycache__/brain_mechanize.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/astroid/brain/__pycache__/brain_multiprocessing.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/astroid/brain/__pycache__/brain_multiprocessing.cpython-39.pyc new file mode 100644 index 0000000..2360ffb Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/astroid/brain/__pycache__/brain_multiprocessing.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/astroid/brain/__pycache__/brain_namedtuple_enum.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/astroid/brain/__pycache__/brain_namedtuple_enum.cpython-39.pyc new file mode 100644 index 0000000..264c0f5 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/astroid/brain/__pycache__/brain_namedtuple_enum.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/astroid/brain/__pycache__/brain_nose.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/astroid/brain/__pycache__/brain_nose.cpython-39.pyc new file mode 100644 index 0000000..763d3e9 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/astroid/brain/__pycache__/brain_nose.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/astroid/brain/__pycache__/brain_numpy_core_fromnumeric.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/astroid/brain/__pycache__/brain_numpy_core_fromnumeric.cpython-39.pyc new file mode 100644 index 0000000..1a7c078 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/astroid/brain/__pycache__/brain_numpy_core_fromnumeric.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/astroid/brain/__pycache__/brain_numpy_core_function_base.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/astroid/brain/__pycache__/brain_numpy_core_function_base.cpython-39.pyc new file mode 100644 index 0000000..c6b47b1 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/astroid/brain/__pycache__/brain_numpy_core_function_base.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/astroid/brain/__pycache__/brain_numpy_core_multiarray.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/astroid/brain/__pycache__/brain_numpy_core_multiarray.cpython-39.pyc new file mode 100644 index 0000000..4bdb3f5 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/astroid/brain/__pycache__/brain_numpy_core_multiarray.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/astroid/brain/__pycache__/brain_numpy_core_numeric.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/astroid/brain/__pycache__/brain_numpy_core_numeric.cpython-39.pyc new file mode 100644 index 0000000..73cc58f Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/astroid/brain/__pycache__/brain_numpy_core_numeric.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/astroid/brain/__pycache__/brain_numpy_core_numerictypes.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/astroid/brain/__pycache__/brain_numpy_core_numerictypes.cpython-39.pyc new file mode 100644 index 0000000..d409c9d Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/astroid/brain/__pycache__/brain_numpy_core_numerictypes.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/astroid/brain/__pycache__/brain_numpy_core_umath.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/astroid/brain/__pycache__/brain_numpy_core_umath.cpython-39.pyc new file mode 100644 index 0000000..0cc82e5 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/astroid/brain/__pycache__/brain_numpy_core_umath.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/astroid/brain/__pycache__/brain_numpy_ndarray.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/astroid/brain/__pycache__/brain_numpy_ndarray.cpython-39.pyc new file mode 100644 index 0000000..5ce1264 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/astroid/brain/__pycache__/brain_numpy_ndarray.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/astroid/brain/__pycache__/brain_numpy_random_mtrand.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/astroid/brain/__pycache__/brain_numpy_random_mtrand.cpython-39.pyc new file mode 100644 index 0000000..dadd87b Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/astroid/brain/__pycache__/brain_numpy_random_mtrand.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/astroid/brain/__pycache__/brain_numpy_utils.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/astroid/brain/__pycache__/brain_numpy_utils.cpython-39.pyc new file mode 100644 index 0000000..068e362 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/astroid/brain/__pycache__/brain_numpy_utils.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/astroid/brain/__pycache__/brain_pkg_resources.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/astroid/brain/__pycache__/brain_pkg_resources.cpython-39.pyc new file mode 100644 index 0000000..02901db Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/astroid/brain/__pycache__/brain_pkg_resources.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/astroid/brain/__pycache__/brain_pytest.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/astroid/brain/__pycache__/brain_pytest.cpython-39.pyc new file mode 100644 index 0000000..51df83a Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/astroid/brain/__pycache__/brain_pytest.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/astroid/brain/__pycache__/brain_qt.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/astroid/brain/__pycache__/brain_qt.cpython-39.pyc new file mode 100644 index 0000000..e53d90d Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/astroid/brain/__pycache__/brain_qt.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/astroid/brain/__pycache__/brain_random.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/astroid/brain/__pycache__/brain_random.cpython-39.pyc new file mode 100644 index 0000000..e6ffd50 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/astroid/brain/__pycache__/brain_random.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/astroid/brain/__pycache__/brain_re.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/astroid/brain/__pycache__/brain_re.cpython-39.pyc new file mode 100644 index 0000000..df24323 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/astroid/brain/__pycache__/brain_re.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/astroid/brain/__pycache__/brain_responses.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/astroid/brain/__pycache__/brain_responses.cpython-39.pyc new file mode 100644 index 0000000..22f2754 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/astroid/brain/__pycache__/brain_responses.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/astroid/brain/__pycache__/brain_scipy_signal.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/astroid/brain/__pycache__/brain_scipy_signal.cpython-39.pyc new file mode 100644 index 0000000..6a6c2ec Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/astroid/brain/__pycache__/brain_scipy_signal.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/astroid/brain/__pycache__/brain_six.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/astroid/brain/__pycache__/brain_six.cpython-39.pyc new file mode 100644 index 0000000..c7cff64 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/astroid/brain/__pycache__/brain_six.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/astroid/brain/__pycache__/brain_ssl.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/astroid/brain/__pycache__/brain_ssl.cpython-39.pyc new file mode 100644 index 0000000..6151b72 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/astroid/brain/__pycache__/brain_ssl.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/astroid/brain/__pycache__/brain_subprocess.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/astroid/brain/__pycache__/brain_subprocess.cpython-39.pyc new file mode 100644 index 0000000..44f7b96 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/astroid/brain/__pycache__/brain_subprocess.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/astroid/brain/__pycache__/brain_threading.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/astroid/brain/__pycache__/brain_threading.cpython-39.pyc new file mode 100644 index 0000000..d4cd008 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/astroid/brain/__pycache__/brain_threading.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/astroid/brain/__pycache__/brain_typing.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/astroid/brain/__pycache__/brain_typing.cpython-39.pyc new file mode 100644 index 0000000..1ceac27 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/astroid/brain/__pycache__/brain_typing.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/astroid/brain/__pycache__/brain_uuid.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/astroid/brain/__pycache__/brain_uuid.cpython-39.pyc new file mode 100644 index 0000000..15fe79a Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/astroid/brain/__pycache__/brain_uuid.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/astroid/brain/brain_argparse.py b/WebCrawler/venv/Lib/site-packages/astroid/brain/brain_argparse.py new file mode 100644 index 0000000..6a7556f --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/astroid/brain/brain_argparse.py @@ -0,0 +1,33 @@ +from astroid import MANAGER, arguments, nodes, inference_tip, UseInferenceDefault + + +def infer_namespace(node, context=None): + callsite = arguments.CallSite.from_call(node, context=context) + if not callsite.keyword_arguments: + # Cannot make sense of it. + raise UseInferenceDefault() + + class_node = nodes.ClassDef("Namespace", "docstring") + class_node.parent = node.parent + for attr in set(callsite.keyword_arguments): + fake_node = nodes.EmptyNode() + fake_node.parent = class_node + fake_node.attrname = attr + class_node.instance_attrs[attr] = [fake_node] + return iter((class_node.instantiate_class(),)) + + +def _looks_like_namespace(node): + func = node.func + if isinstance(func, nodes.Attribute): + return ( + func.attrname == "Namespace" + and isinstance(func.expr, nodes.Name) + and func.expr.name == "argparse" + ) + return False + + +MANAGER.register_transform( + nodes.Call, inference_tip(infer_namespace), _looks_like_namespace +) diff --git a/WebCrawler/venv/Lib/site-packages/astroid/brain/brain_attrs.py b/WebCrawler/venv/Lib/site-packages/astroid/brain/brain_attrs.py new file mode 100644 index 0000000..670736f --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/astroid/brain/brain_attrs.py @@ -0,0 +1,65 @@ +# Licensed under the LGPL: https://www.gnu.org/licenses/old-licenses/lgpl-2.1.en.html +# For details: https://github.com/PyCQA/astroid/blob/master/COPYING.LESSER +""" +Astroid hook for the attrs library + +Without this hook pylint reports unsupported-assignment-operation +for attrs classes +""" + +import astroid +from astroid import MANAGER + + +ATTRIB_NAMES = frozenset(("attr.ib", "attrib", "attr.attrib")) +ATTRS_NAMES = frozenset(("attr.s", "attrs", "attr.attrs", "attr.attributes")) + + +def is_decorated_with_attrs(node, decorator_names=ATTRS_NAMES): + """Return True if a decorated node has + an attr decorator applied.""" + if not node.decorators: + return False + for decorator_attribute in node.decorators.nodes: + if isinstance(decorator_attribute, astroid.Call): # decorator with arguments + decorator_attribute = decorator_attribute.func + if decorator_attribute.as_string() in decorator_names: + return True + return False + + +def attr_attributes_transform(node): + """Given that the ClassNode has an attr decorator, + rewrite class attributes as instance attributes + """ + # Astroid can't infer this attribute properly + # Prevents https://github.com/PyCQA/pylint/issues/1884 + node.locals["__attrs_attrs__"] = [astroid.Unknown(parent=node)] + + for cdefbodynode in node.body: + if not isinstance(cdefbodynode, (astroid.Assign, astroid.AnnAssign)): + continue + if isinstance(cdefbodynode.value, astroid.Call): + if cdefbodynode.value.func.as_string() not in ATTRIB_NAMES: + continue + else: + continue + targets = ( + cdefbodynode.targets + if hasattr(cdefbodynode, "targets") + else [cdefbodynode.target] + ) + for target in targets: + + rhs_node = astroid.Unknown( + lineno=cdefbodynode.lineno, + col_offset=cdefbodynode.col_offset, + parent=cdefbodynode, + ) + node.locals[target.name] = [rhs_node] + node.instance_attrs[target.name] = [rhs_node] + + +MANAGER.register_transform( + astroid.ClassDef, attr_attributes_transform, is_decorated_with_attrs +) diff --git a/WebCrawler/venv/Lib/site-packages/astroid/brain/brain_boto3.py b/WebCrawler/venv/Lib/site-packages/astroid/brain/brain_boto3.py new file mode 100644 index 0000000..342ca57 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/astroid/brain/brain_boto3.py @@ -0,0 +1,28 @@ +# Licensed under the LGPL: https://www.gnu.org/licenses/old-licenses/lgpl-2.1.en.html +# For details: https://github.com/PyCQA/astroid/blob/master/COPYING.LESSER + +"""Astroid hooks for understanding boto3.ServiceRequest()""" +import astroid +from astroid import MANAGER, extract_node + +BOTO_SERVICE_FACTORY_QUALIFIED_NAME = "boto3.resources.base.ServiceResource" + + +def service_request_transform(node): + """Transform ServiceResource to look like dynamic classes""" + code = """ + def __getattr__(self, attr): + return 0 + """ + func_getattr = extract_node(code) + node.locals["__getattr__"] = [func_getattr] + return node + + +def _looks_like_boto3_service_request(node): + return node.qname() == BOTO_SERVICE_FACTORY_QUALIFIED_NAME + + +MANAGER.register_transform( + astroid.ClassDef, service_request_transform, _looks_like_boto3_service_request +) diff --git a/WebCrawler/venv/Lib/site-packages/astroid/brain/brain_builtin_inference.py b/WebCrawler/venv/Lib/site-packages/astroid/brain/brain_builtin_inference.py new file mode 100644 index 0000000..4b07ac5 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/astroid/brain/brain_builtin_inference.py @@ -0,0 +1,873 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2014-2020 Claudiu Popa +# Copyright (c) 2014-2015 LOGILAB S.A. (Paris, FRANCE) +# Copyright (c) 2015-2016 Ceridwen +# Copyright (c) 2015 Rene Zhang +# Copyright (c) 2018 Bryce Guinta +# Copyright (c) 2018 Ville Skyttä +# Copyright (c) 2019 Stanislav Levin +# Copyright (c) 2019 David Liu +# Copyright (c) 2019 Bryce Guinta +# Copyright (c) 2019 Frédéric Chapoton + +# Licensed under the LGPL: https://www.gnu.org/licenses/old-licenses/lgpl-2.1.en.html +# For details: https://github.com/PyCQA/astroid/blob/master/COPYING.LESSER + +"""Astroid hooks for various builtins.""" + +from functools import partial +from textwrap import dedent + +import six +from astroid import ( + MANAGER, + UseInferenceDefault, + AttributeInferenceError, + inference_tip, + InferenceError, + NameInferenceError, + AstroidTypeError, + MroError, +) +from astroid import arguments +from astroid.builder import AstroidBuilder +from astroid import helpers +from astroid import nodes +from astroid import objects +from astroid import scoped_nodes +from astroid import util + + +OBJECT_DUNDER_NEW = "object.__new__" + + +def _extend_str(class_node, rvalue): + """function to extend builtin str/unicode class""" + code = dedent( + """ + class whatever(object): + def join(self, iterable): + return {rvalue} + def replace(self, old, new, count=None): + return {rvalue} + def format(self, *args, **kwargs): + return {rvalue} + def encode(self, encoding='ascii', errors=None): + return '' + def decode(self, encoding='ascii', errors=None): + return u'' + def capitalize(self): + return {rvalue} + def title(self): + return {rvalue} + def lower(self): + return {rvalue} + def upper(self): + return {rvalue} + def swapcase(self): + return {rvalue} + def index(self, sub, start=None, end=None): + return 0 + def find(self, sub, start=None, end=None): + return 0 + def count(self, sub, start=None, end=None): + return 0 + def strip(self, chars=None): + return {rvalue} + def lstrip(self, chars=None): + return {rvalue} + def rstrip(self, chars=None): + return {rvalue} + def rjust(self, width, fillchar=None): + return {rvalue} + def center(self, width, fillchar=None): + return {rvalue} + def ljust(self, width, fillchar=None): + return {rvalue} + """ + ) + code = code.format(rvalue=rvalue) + fake = AstroidBuilder(MANAGER).string_build(code)["whatever"] + for method in fake.mymethods(): + method.parent = class_node + method.lineno = None + method.col_offset = None + if "__class__" in method.locals: + method.locals["__class__"] = [class_node] + class_node.locals[method.name] = [method] + method.parent = class_node + + +def _extend_builtins(class_transforms): + builtin_ast = MANAGER.builtins_module + for class_name, transform in class_transforms.items(): + transform(builtin_ast[class_name]) + + +_extend_builtins( + { + "bytes": partial(_extend_str, rvalue="b''"), + "str": partial(_extend_str, rvalue="''"), + } +) + + +def _builtin_filter_predicate(node, builtin_name): + if isinstance(node.func, nodes.Name) and node.func.name == builtin_name: + return True + if isinstance(node.func, nodes.Attribute): + return ( + node.func.attrname == "fromkeys" + and isinstance(node.func.expr, nodes.Name) + and node.func.expr.name == "dict" + ) + return False + + +def register_builtin_transform(transform, builtin_name): + """Register a new transform function for the given *builtin_name*. + + The transform function must accept two parameters, a node and + an optional context. + """ + + def _transform_wrapper(node, context=None): + result = transform(node, context=context) + if result: + if not result.parent: + # Let the transformation function determine + # the parent for its result. Otherwise, + # we set it to be the node we transformed from. + result.parent = node + + if result.lineno is None: + result.lineno = node.lineno + if result.col_offset is None: + result.col_offset = node.col_offset + return iter([result]) + + MANAGER.register_transform( + nodes.Call, + inference_tip(_transform_wrapper), + partial(_builtin_filter_predicate, builtin_name=builtin_name), + ) + + +def _container_generic_inference(node, context, node_type, transform): + args = node.args + if not args: + return node_type() + if len(node.args) > 1: + raise UseInferenceDefault() + + (arg,) = args + transformed = transform(arg) + if not transformed: + try: + inferred = next(arg.infer(context=context)) + except (InferenceError, StopIteration): + raise UseInferenceDefault() + if inferred is util.Uninferable: + raise UseInferenceDefault() + transformed = transform(inferred) + if not transformed or transformed is util.Uninferable: + raise UseInferenceDefault() + return transformed + + +def _container_generic_transform(arg, context, klass, iterables, build_elts): + if isinstance(arg, klass): + return arg + elif isinstance(arg, iterables): + if all(isinstance(elt, nodes.Const) for elt in arg.elts): + elts = [elt.value for elt in arg.elts] + else: + # TODO: Does not handle deduplication for sets. + elts = [] + for element in arg.elts: + inferred = helpers.safe_infer(element, context=context) + if inferred: + evaluated_object = nodes.EvaluatedObject( + original=element, value=inferred + ) + elts.append(evaluated_object) + elif isinstance(arg, nodes.Dict): + # Dicts need to have consts as strings already. + if not all(isinstance(elt[0], nodes.Const) for elt in arg.items): + raise UseInferenceDefault() + elts = [item[0].value for item in arg.items] + elif isinstance(arg, nodes.Const) and isinstance( + arg.value, (six.string_types, six.binary_type) + ): + elts = arg.value + else: + return + return klass.from_elements(elts=build_elts(elts)) + + +def _infer_builtin_container( + node, context, klass=None, iterables=None, build_elts=None +): + transform_func = partial( + _container_generic_transform, + context=context, + klass=klass, + iterables=iterables, + build_elts=build_elts, + ) + + return _container_generic_inference(node, context, klass, transform_func) + + +# pylint: disable=invalid-name +infer_tuple = partial( + _infer_builtin_container, + klass=nodes.Tuple, + iterables=( + nodes.List, + nodes.Set, + objects.FrozenSet, + objects.DictItems, + objects.DictKeys, + objects.DictValues, + ), + build_elts=tuple, +) + +infer_list = partial( + _infer_builtin_container, + klass=nodes.List, + iterables=( + nodes.Tuple, + nodes.Set, + objects.FrozenSet, + objects.DictItems, + objects.DictKeys, + objects.DictValues, + ), + build_elts=list, +) + +infer_set = partial( + _infer_builtin_container, + klass=nodes.Set, + iterables=(nodes.List, nodes.Tuple, objects.FrozenSet, objects.DictKeys), + build_elts=set, +) + +infer_frozenset = partial( + _infer_builtin_container, + klass=objects.FrozenSet, + iterables=(nodes.List, nodes.Tuple, nodes.Set, objects.FrozenSet, objects.DictKeys), + build_elts=frozenset, +) + + +def _get_elts(arg, context): + is_iterable = lambda n: isinstance(n, (nodes.List, nodes.Tuple, nodes.Set)) + try: + inferred = next(arg.infer(context)) + except (InferenceError, NameInferenceError): + raise UseInferenceDefault() + if isinstance(inferred, nodes.Dict): + items = inferred.items + elif is_iterable(inferred): + items = [] + for elt in inferred.elts: + # If an item is not a pair of two items, + # then fallback to the default inference. + # Also, take in consideration only hashable items, + # tuples and consts. We are choosing Names as well. + if not is_iterable(elt): + raise UseInferenceDefault() + if len(elt.elts) != 2: + raise UseInferenceDefault() + if not isinstance(elt.elts[0], (nodes.Tuple, nodes.Const, nodes.Name)): + raise UseInferenceDefault() + items.append(tuple(elt.elts)) + else: + raise UseInferenceDefault() + return items + + +def infer_dict(node, context=None): + """Try to infer a dict call to a Dict node. + + The function treats the following cases: + + * dict() + * dict(mapping) + * dict(iterable) + * dict(iterable, **kwargs) + * dict(mapping, **kwargs) + * dict(**kwargs) + + If a case can't be inferred, we'll fallback to default inference. + """ + call = arguments.CallSite.from_call(node, context=context) + if call.has_invalid_arguments() or call.has_invalid_keywords(): + raise UseInferenceDefault + + args = call.positional_arguments + kwargs = list(call.keyword_arguments.items()) + + if not args and not kwargs: + # dict() + return nodes.Dict() + elif kwargs and not args: + # dict(a=1, b=2, c=4) + items = [(nodes.Const(key), value) for key, value in kwargs] + elif len(args) == 1 and kwargs: + # dict(some_iterable, b=2, c=4) + elts = _get_elts(args[0], context) + keys = [(nodes.Const(key), value) for key, value in kwargs] + items = elts + keys + elif len(args) == 1: + items = _get_elts(args[0], context) + else: + raise UseInferenceDefault() + + value = nodes.Dict( + col_offset=node.col_offset, lineno=node.lineno, parent=node.parent + ) + value.postinit(items) + return value + + +def infer_super(node, context=None): + """Understand super calls. + + There are some restrictions for what can be understood: + + * unbounded super (one argument form) is not understood. + + * if the super call is not inside a function (classmethod or method), + then the default inference will be used. + + * if the super arguments can't be inferred, the default inference + will be used. + """ + if len(node.args) == 1: + # Ignore unbounded super. + raise UseInferenceDefault + + scope = node.scope() + if not isinstance(scope, nodes.FunctionDef): + # Ignore non-method uses of super. + raise UseInferenceDefault + if scope.type not in ("classmethod", "method"): + # Not interested in staticmethods. + raise UseInferenceDefault + + cls = scoped_nodes.get_wrapping_class(scope) + if not len(node.args): + mro_pointer = cls + # In we are in a classmethod, the interpreter will fill + # automatically the class as the second argument, not an instance. + if scope.type == "classmethod": + mro_type = cls + else: + mro_type = cls.instantiate_class() + else: + try: + mro_pointer = next(node.args[0].infer(context=context)) + except InferenceError: + raise UseInferenceDefault + try: + mro_type = next(node.args[1].infer(context=context)) + except InferenceError: + raise UseInferenceDefault + + if mro_pointer is util.Uninferable or mro_type is util.Uninferable: + # No way we could understand this. + raise UseInferenceDefault + + super_obj = objects.Super( + mro_pointer=mro_pointer, mro_type=mro_type, self_class=cls, scope=scope + ) + super_obj.parent = node + return super_obj + + +def _infer_getattr_args(node, context): + if len(node.args) not in (2, 3): + # Not a valid getattr call. + raise UseInferenceDefault + + try: + obj = next(node.args[0].infer(context=context)) + attr = next(node.args[1].infer(context=context)) + except InferenceError: + raise UseInferenceDefault + + if obj is util.Uninferable or attr is util.Uninferable: + # If one of the arguments is something we can't infer, + # then also make the result of the getattr call something + # which is unknown. + return util.Uninferable, util.Uninferable + + is_string = isinstance(attr, nodes.Const) and isinstance( + attr.value, six.string_types + ) + if not is_string: + raise UseInferenceDefault + + return obj, attr.value + + +def infer_getattr(node, context=None): + """Understand getattr calls + + If one of the arguments is an Uninferable object, then the + result will be an Uninferable object. Otherwise, the normal attribute + lookup will be done. + """ + obj, attr = _infer_getattr_args(node, context) + if ( + obj is util.Uninferable + or attr is util.Uninferable + or not hasattr(obj, "igetattr") + ): + return util.Uninferable + + try: + return next(obj.igetattr(attr, context=context)) + except (StopIteration, InferenceError, AttributeInferenceError): + if len(node.args) == 3: + # Try to infer the default and return it instead. + try: + return next(node.args[2].infer(context=context)) + except InferenceError: + raise UseInferenceDefault + + raise UseInferenceDefault + + +def infer_hasattr(node, context=None): + """Understand hasattr calls + + This always guarantees three possible outcomes for calling + hasattr: Const(False) when we are sure that the object + doesn't have the intended attribute, Const(True) when + we know that the object has the attribute and Uninferable + when we are unsure of the outcome of the function call. + """ + try: + obj, attr = _infer_getattr_args(node, context) + if ( + obj is util.Uninferable + or attr is util.Uninferable + or not hasattr(obj, "getattr") + ): + return util.Uninferable + obj.getattr(attr, context=context) + except UseInferenceDefault: + # Can't infer something from this function call. + return util.Uninferable + except AttributeInferenceError: + # Doesn't have it. + return nodes.Const(False) + return nodes.Const(True) + + +def infer_callable(node, context=None): + """Understand callable calls + + This follows Python's semantics, where an object + is callable if it provides an attribute __call__, + even though that attribute is something which can't be + called. + """ + if len(node.args) != 1: + # Invalid callable call. + raise UseInferenceDefault + + argument = node.args[0] + try: + inferred = next(argument.infer(context=context)) + except InferenceError: + return util.Uninferable + if inferred is util.Uninferable: + return util.Uninferable + return nodes.Const(inferred.callable()) + + +def infer_property(node, context=None): + """Understand `property` class + + This only infers the output of `property` + call, not the arguments themselves. + """ + if len(node.args) < 1: + # Invalid property call. + raise UseInferenceDefault + + getter = node.args[0] + try: + inferred = next(getter.infer(context=context)) + except InferenceError: + raise UseInferenceDefault + + if not isinstance(inferred, (nodes.FunctionDef, nodes.Lambda)): + raise UseInferenceDefault + + return objects.Property( + function=inferred, + name=inferred.name, + doc=getattr(inferred, "doc", None), + lineno=node.lineno, + parent=node, + col_offset=node.col_offset, + ) + + +def infer_bool(node, context=None): + """Understand bool calls.""" + if len(node.args) > 1: + # Invalid bool call. + raise UseInferenceDefault + + if not node.args: + return nodes.Const(False) + + argument = node.args[0] + try: + inferred = next(argument.infer(context=context)) + except InferenceError: + return util.Uninferable + if inferred is util.Uninferable: + return util.Uninferable + + bool_value = inferred.bool_value(context=context) + if bool_value is util.Uninferable: + return util.Uninferable + return nodes.Const(bool_value) + + +def infer_type(node, context=None): + """Understand the one-argument form of *type*.""" + if len(node.args) != 1: + raise UseInferenceDefault + + return helpers.object_type(node.args[0], context) + + +def infer_slice(node, context=None): + """Understand `slice` calls.""" + args = node.args + if not 0 < len(args) <= 3: + raise UseInferenceDefault + + infer_func = partial(helpers.safe_infer, context=context) + args = [infer_func(arg) for arg in args] + for arg in args: + if not arg or arg is util.Uninferable: + raise UseInferenceDefault + if not isinstance(arg, nodes.Const): + raise UseInferenceDefault + if not isinstance(arg.value, (type(None), int)): + raise UseInferenceDefault + + if len(args) < 3: + # Make sure we have 3 arguments. + args.extend([None] * (3 - len(args))) + + slice_node = nodes.Slice( + lineno=node.lineno, col_offset=node.col_offset, parent=node.parent + ) + slice_node.postinit(*args) + return slice_node + + +def _infer_object__new__decorator(node, context=None): + # Instantiate class immediately + # since that's what @object.__new__ does + return iter((node.instantiate_class(),)) + + +def _infer_object__new__decorator_check(node): + """Predicate before inference_tip + + Check if the given ClassDef has an @object.__new__ decorator + """ + if not node.decorators: + return False + + for decorator in node.decorators.nodes: + if isinstance(decorator, nodes.Attribute): + if decorator.as_string() == OBJECT_DUNDER_NEW: + return True + return False + + +def infer_issubclass(callnode, context=None): + """Infer issubclass() calls + + :param nodes.Call callnode: an `issubclass` call + :param InferenceContext: the context for the inference + :rtype nodes.Const: Boolean Const value of the `issubclass` call + :raises UseInferenceDefault: If the node cannot be inferred + """ + call = arguments.CallSite.from_call(callnode, context=context) + if call.keyword_arguments: + # issubclass doesn't support keyword arguments + raise UseInferenceDefault("TypeError: issubclass() takes no keyword arguments") + if len(call.positional_arguments) != 2: + raise UseInferenceDefault( + "Expected two arguments, got {count}".format( + count=len(call.positional_arguments) + ) + ) + # The left hand argument is the obj to be checked + obj_node, class_or_tuple_node = call.positional_arguments + + try: + obj_type = next(obj_node.infer(context=context)) + except InferenceError as exc: + raise UseInferenceDefault from exc + if not isinstance(obj_type, nodes.ClassDef): + raise UseInferenceDefault("TypeError: arg 1 must be class") + + # The right hand argument is the class(es) that the given + # object is to be checked against. + try: + class_container = _class_or_tuple_to_container( + class_or_tuple_node, context=context + ) + except InferenceError as exc: + raise UseInferenceDefault from exc + try: + issubclass_bool = helpers.object_issubclass(obj_type, class_container, context) + except AstroidTypeError as exc: + raise UseInferenceDefault("TypeError: " + str(exc)) from exc + except MroError as exc: + raise UseInferenceDefault from exc + return nodes.Const(issubclass_bool) + + +def infer_isinstance(callnode, context=None): + """Infer isinstance calls + + :param nodes.Call callnode: an isinstance call + :param InferenceContext: context for call + (currently unused but is a common interface for inference) + :rtype nodes.Const: Boolean Const value of isinstance call + + :raises UseInferenceDefault: If the node cannot be inferred + """ + call = arguments.CallSite.from_call(callnode, context=context) + if call.keyword_arguments: + # isinstance doesn't support keyword arguments + raise UseInferenceDefault("TypeError: isinstance() takes no keyword arguments") + if len(call.positional_arguments) != 2: + raise UseInferenceDefault( + "Expected two arguments, got {count}".format( + count=len(call.positional_arguments) + ) + ) + # The left hand argument is the obj to be checked + obj_node, class_or_tuple_node = call.positional_arguments + # The right hand argument is the class(es) that the given + # obj is to be check is an instance of + try: + class_container = _class_or_tuple_to_container( + class_or_tuple_node, context=context + ) + except InferenceError: + raise UseInferenceDefault + try: + isinstance_bool = helpers.object_isinstance(obj_node, class_container, context) + except AstroidTypeError as exc: + raise UseInferenceDefault("TypeError: " + str(exc)) + except MroError as exc: + raise UseInferenceDefault from exc + if isinstance_bool is util.Uninferable: + raise UseInferenceDefault + return nodes.Const(isinstance_bool) + + +def _class_or_tuple_to_container(node, context=None): + # Move inferences results into container + # to simplify later logic + # raises InferenceError if any of the inferences fall through + node_infer = next(node.infer(context=context)) + # arg2 MUST be a type or a TUPLE of types + # for isinstance + if isinstance(node_infer, nodes.Tuple): + class_container = [ + next(node.infer(context=context)) for node in node_infer.elts + ] + class_container = [ + klass_node for klass_node in class_container if klass_node is not None + ] + else: + class_container = [node_infer] + return class_container + + +def infer_len(node, context=None): + """Infer length calls + + :param nodes.Call node: len call to infer + :param context.InferenceContext: node context + :rtype nodes.Const: a Const node with the inferred length, if possible + """ + call = arguments.CallSite.from_call(node, context=context) + if call.keyword_arguments: + raise UseInferenceDefault("TypeError: len() must take no keyword arguments") + if len(call.positional_arguments) != 1: + raise UseInferenceDefault( + "TypeError: len() must take exactly one argument " + "({len}) given".format(len=len(call.positional_arguments)) + ) + [argument_node] = call.positional_arguments + try: + return nodes.Const(helpers.object_len(argument_node, context=context)) + except (AstroidTypeError, InferenceError) as exc: + raise UseInferenceDefault(str(exc)) from exc + + +def infer_str(node, context=None): + """Infer str() calls + + :param nodes.Call node: str() call to infer + :param context.InferenceContext: node context + :rtype nodes.Const: a Const containing an empty string + """ + call = arguments.CallSite.from_call(node, context=context) + if call.keyword_arguments: + raise UseInferenceDefault("TypeError: str() must take no keyword arguments") + try: + return nodes.Const("") + except (AstroidTypeError, InferenceError) as exc: + raise UseInferenceDefault(str(exc)) from exc + + +def infer_int(node, context=None): + """Infer int() calls + + :param nodes.Call node: int() call to infer + :param context.InferenceContext: node context + :rtype nodes.Const: a Const containing the integer value of the int() call + """ + call = arguments.CallSite.from_call(node, context=context) + if call.keyword_arguments: + raise UseInferenceDefault("TypeError: int() must take no keyword arguments") + + if call.positional_arguments: + try: + first_value = next(call.positional_arguments[0].infer(context=context)) + except (InferenceError, StopIteration) as exc: + raise UseInferenceDefault(str(exc)) from exc + + if first_value is util.Uninferable: + raise UseInferenceDefault + + if isinstance(first_value, nodes.Const) and isinstance( + first_value.value, (int, str) + ): + try: + actual_value = int(first_value.value) + except ValueError: + return nodes.Const(0) + return nodes.Const(actual_value) + + return nodes.Const(0) + + +def infer_dict_fromkeys(node, context=None): + """Infer dict.fromkeys + + :param nodes.Call node: dict.fromkeys() call to infer + :param context.InferenceContext: node context + :rtype nodes.Dict: + a Dictionary containing the values that astroid was able to infer. + In case the inference failed for any reason, an empty dictionary + will be inferred instead. + """ + + def _build_dict_with_elements(elements): + new_node = nodes.Dict( + col_offset=node.col_offset, lineno=node.lineno, parent=node.parent + ) + new_node.postinit(elements) + return new_node + + call = arguments.CallSite.from_call(node, context=context) + if call.keyword_arguments: + raise UseInferenceDefault("TypeError: int() must take no keyword arguments") + if len(call.positional_arguments) not in {1, 2}: + raise UseInferenceDefault( + "TypeError: Needs between 1 and 2 positional arguments" + ) + + default = nodes.Const(None) + values = call.positional_arguments[0] + try: + inferred_values = next(values.infer(context=context)) + except InferenceError: + return _build_dict_with_elements([]) + if inferred_values is util.Uninferable: + return _build_dict_with_elements([]) + + # Limit to a couple of potential values, as this can become pretty complicated + accepted_iterable_elements = (nodes.Const,) + if isinstance(inferred_values, (nodes.List, nodes.Set, nodes.Tuple)): + elements = inferred_values.elts + for element in elements: + if not isinstance(element, accepted_iterable_elements): + # Fallback to an empty dict + return _build_dict_with_elements([]) + + elements_with_value = [(element, default) for element in elements] + return _build_dict_with_elements(elements_with_value) + + elif isinstance(inferred_values, nodes.Const) and isinstance( + inferred_values.value, (str, bytes) + ): + elements = [ + (nodes.Const(element), default) for element in inferred_values.value + ] + return _build_dict_with_elements(elements) + elif isinstance(inferred_values, nodes.Dict): + keys = inferred_values.itered() + for key in keys: + if not isinstance(key, accepted_iterable_elements): + # Fallback to an empty dict + return _build_dict_with_elements([]) + + elements_with_value = [(element, default) for element in keys] + return _build_dict_with_elements(elements_with_value) + + # Fallback to an empty dictionary + return _build_dict_with_elements([]) + + +# Builtins inference +register_builtin_transform(infer_bool, "bool") +register_builtin_transform(infer_super, "super") +register_builtin_transform(infer_callable, "callable") +register_builtin_transform(infer_property, "property") +register_builtin_transform(infer_getattr, "getattr") +register_builtin_transform(infer_hasattr, "hasattr") +register_builtin_transform(infer_tuple, "tuple") +register_builtin_transform(infer_set, "set") +register_builtin_transform(infer_list, "list") +register_builtin_transform(infer_dict, "dict") +register_builtin_transform(infer_frozenset, "frozenset") +register_builtin_transform(infer_type, "type") +register_builtin_transform(infer_slice, "slice") +register_builtin_transform(infer_isinstance, "isinstance") +register_builtin_transform(infer_issubclass, "issubclass") +register_builtin_transform(infer_len, "len") +register_builtin_transform(infer_str, "str") +register_builtin_transform(infer_int, "int") +register_builtin_transform(infer_dict_fromkeys, "dict.fromkeys") + + +# Infer object.__new__ calls +MANAGER.register_transform( + nodes.ClassDef, + inference_tip(_infer_object__new__decorator), + _infer_object__new__decorator_check, +) diff --git a/WebCrawler/venv/Lib/site-packages/astroid/brain/brain_collections.py b/WebCrawler/venv/Lib/site-packages/astroid/brain/brain_collections.py new file mode 100644 index 0000000..669c6ca --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/astroid/brain/brain_collections.py @@ -0,0 +1,75 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2016, 2018 Claudiu Popa +# Copyright (c) 2016-2017 Łukasz Rogalski +# Copyright (c) 2017 Derek Gustafson +# Copyright (c) 2018 Ioana Tagirta +# Copyright (c) 2019 Hugo van Kemenade + +# Licensed under the LGPL: https://www.gnu.org/licenses/old-licenses/lgpl-2.1.en.html +# For details: https://github.com/PyCQA/astroid/blob/master/COPYING.LESSER +import sys + +import astroid + + +def _collections_transform(): + return astroid.parse( + """ + class defaultdict(dict): + default_factory = None + def __missing__(self, key): pass + def __getitem__(self, key): return default_factory + + """ + + _deque_mock() + + _ordered_dict_mock() + ) + + +def _deque_mock(): + base_deque_class = """ + class deque(object): + maxlen = 0 + def __init__(self, iterable=None, maxlen=None): + self.iterable = iterable or [] + def append(self, x): pass + def appendleft(self, x): pass + def clear(self): pass + def count(self, x): return 0 + def extend(self, iterable): pass + def extendleft(self, iterable): pass + def pop(self): return self.iterable[0] + def popleft(self): return self.iterable[0] + def remove(self, value): pass + def reverse(self): return reversed(self.iterable) + def rotate(self, n=1): return self + def __iter__(self): return self + def __reversed__(self): return self.iterable[::-1] + def __getitem__(self, index): return self.iterable[index] + def __setitem__(self, index, value): pass + def __delitem__(self, index): pass + def __bool__(self): return bool(self.iterable) + def __nonzero__(self): return bool(self.iterable) + def __contains__(self, o): return o in self.iterable + def __len__(self): return len(self.iterable) + def __copy__(self): return deque(self.iterable) + def copy(self): return deque(self.iterable) + def index(self, x, start=0, end=0): return 0 + def insert(self, x, i): pass + def __add__(self, other): pass + def __iadd__(self, other): pass + def __mul__(self, other): pass + def __imul__(self, other): pass + def __rmul__(self, other): pass""" + return base_deque_class + + +def _ordered_dict_mock(): + base_ordered_dict_class = """ + class OrderedDict(dict): + def __reversed__(self): return self[::-1] + def move_to_end(self, key, last=False): pass""" + return base_ordered_dict_class + + +astroid.register_module_extender(astroid.MANAGER, "collections", _collections_transform) diff --git a/WebCrawler/venv/Lib/site-packages/astroid/brain/brain_crypt.py b/WebCrawler/venv/Lib/site-packages/astroid/brain/brain_crypt.py new file mode 100644 index 0000000..491ee23 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/astroid/brain/brain_crypt.py @@ -0,0 +1,26 @@ +# Licensed under the LGPL: https://www.gnu.org/licenses/old-licenses/lgpl-2.1.en.html +# For details: https://github.com/PyCQA/astroid/blob/master/COPYING.LESSER +import sys +import astroid + +PY37 = sys.version_info >= (3, 7) + +if PY37: + # Since Python 3.7 Hashing Methods are added + # dynamically to globals() + + def _re_transform(): + return astroid.parse( + """ + from collections import namedtuple + _Method = namedtuple('_Method', 'name ident salt_chars total_size') + + METHOD_SHA512 = _Method('SHA512', '6', 16, 106) + METHOD_SHA256 = _Method('SHA256', '5', 16, 63) + METHOD_BLOWFISH = _Method('BLOWFISH', 2, 'b', 22) + METHOD_MD5 = _Method('MD5', '1', 8, 34) + METHOD_CRYPT = _Method('CRYPT', None, 2, 13) + """ + ) + + astroid.register_module_extender(astroid.MANAGER, "crypt", _re_transform) diff --git a/WebCrawler/venv/Lib/site-packages/astroid/brain/brain_curses.py b/WebCrawler/venv/Lib/site-packages/astroid/brain/brain_curses.py new file mode 100644 index 0000000..68e88b9 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/astroid/brain/brain_curses.py @@ -0,0 +1,179 @@ +# Licensed under the LGPL: https://www.gnu.org/licenses/old-licenses/lgpl-2.1.en.html +# For details: https://github.com/PyCQA/astroid/blob/master/COPYING.LESSER +import astroid + + +def _curses_transform(): + return astroid.parse( + """ + A_ALTCHARSET = 1 + A_BLINK = 1 + A_BOLD = 1 + A_DIM = 1 + A_INVIS = 1 + A_ITALIC = 1 + A_NORMAL = 1 + A_PROTECT = 1 + A_REVERSE = 1 + A_STANDOUT = 1 + A_UNDERLINE = 1 + A_HORIZONTAL = 1 + A_LEFT = 1 + A_LOW = 1 + A_RIGHT = 1 + A_TOP = 1 + A_VERTICAL = 1 + A_CHARTEXT = 1 + A_ATTRIBUTES = 1 + A_CHARTEXT = 1 + A_COLOR = 1 + KEY_MIN = 1 + KEY_BREAK = 1 + KEY_DOWN = 1 + KEY_UP = 1 + KEY_LEFT = 1 + KEY_RIGHT = 1 + KEY_HOME = 1 + KEY_BACKSPACE = 1 + KEY_F0 = 1 + KEY_Fn = 1 + KEY_DL = 1 + KEY_IL = 1 + KEY_DC = 1 + KEY_IC = 1 + KEY_EIC = 1 + KEY_CLEAR = 1 + KEY_EOS = 1 + KEY_EOL = 1 + KEY_SF = 1 + KEY_SR = 1 + KEY_NPAGE = 1 + KEY_PPAGE = 1 + KEY_STAB = 1 + KEY_CTAB = 1 + KEY_CATAB = 1 + KEY_ENTER = 1 + KEY_SRESET = 1 + KEY_RESET = 1 + KEY_PRINT = 1 + KEY_LL = 1 + KEY_A1 = 1 + KEY_A3 = 1 + KEY_B2 = 1 + KEY_C1 = 1 + KEY_C3 = 1 + KEY_BTAB = 1 + KEY_BEG = 1 + KEY_CANCEL = 1 + KEY_CLOSE = 1 + KEY_COMMAND = 1 + KEY_COPY = 1 + KEY_CREATE = 1 + KEY_END = 1 + KEY_EXIT = 1 + KEY_FIND = 1 + KEY_HELP = 1 + KEY_MARK = 1 + KEY_MESSAGE = 1 + KEY_MOVE = 1 + KEY_NEXT = 1 + KEY_OPEN = 1 + KEY_OPTIONS = 1 + KEY_PREVIOUS = 1 + KEY_REDO = 1 + KEY_REFERENCE = 1 + KEY_REFRESH = 1 + KEY_REPLACE = 1 + KEY_RESTART = 1 + KEY_RESUME = 1 + KEY_SAVE = 1 + KEY_SBEG = 1 + KEY_SCANCEL = 1 + KEY_SCOMMAND = 1 + KEY_SCOPY = 1 + KEY_SCREATE = 1 + KEY_SDC = 1 + KEY_SDL = 1 + KEY_SELECT = 1 + KEY_SEND = 1 + KEY_SEOL = 1 + KEY_SEXIT = 1 + KEY_SFIND = 1 + KEY_SHELP = 1 + KEY_SHOME = 1 + KEY_SIC = 1 + KEY_SLEFT = 1 + KEY_SMESSAGE = 1 + KEY_SMOVE = 1 + KEY_SNEXT = 1 + KEY_SOPTIONS = 1 + KEY_SPREVIOUS = 1 + KEY_SPRINT = 1 + KEY_SREDO = 1 + KEY_SREPLACE = 1 + KEY_SRIGHT = 1 + KEY_SRSUME = 1 + KEY_SSAVE = 1 + KEY_SSUSPEND = 1 + KEY_SUNDO = 1 + KEY_SUSPEND = 1 + KEY_UNDO = 1 + KEY_MOUSE = 1 + KEY_RESIZE = 1 + KEY_MAX = 1 + ACS_BBSS = 1 + ACS_BLOCK = 1 + ACS_BOARD = 1 + ACS_BSBS = 1 + ACS_BSSB = 1 + ACS_BSSS = 1 + ACS_BTEE = 1 + ACS_BULLET = 1 + ACS_CKBOARD = 1 + ACS_DARROW = 1 + ACS_DEGREE = 1 + ACS_DIAMOND = 1 + ACS_GEQUAL = 1 + ACS_HLINE = 1 + ACS_LANTERN = 1 + ACS_LARROW = 1 + ACS_LEQUAL = 1 + ACS_LLCORNER = 1 + ACS_LRCORNER = 1 + ACS_LTEE = 1 + ACS_NEQUAL = 1 + ACS_PI = 1 + ACS_PLMINUS = 1 + ACS_PLUS = 1 + ACS_RARROW = 1 + ACS_RTEE = 1 + ACS_S1 = 1 + ACS_S3 = 1 + ACS_S7 = 1 + ACS_S9 = 1 + ACS_SBBS = 1 + ACS_SBSB = 1 + ACS_SBSS = 1 + ACS_SSBB = 1 + ACS_SSBS = 1 + ACS_SSSB = 1 + ACS_SSSS = 1 + ACS_STERLING = 1 + ACS_TTEE = 1 + ACS_UARROW = 1 + ACS_ULCORNER = 1 + ACS_URCORNER = 1 + ACS_VLINE = 1 + COLOR_BLACK = 1 + COLOR_BLUE = 1 + COLOR_CYAN = 1 + COLOR_GREEN = 1 + COLOR_MAGENTA = 1 + COLOR_RED = 1 + COLOR_WHITE = 1 + COLOR_YELLOW = 1 + """ + ) + + +astroid.register_module_extender(astroid.MANAGER, "curses", _curses_transform) diff --git a/WebCrawler/venv/Lib/site-packages/astroid/brain/brain_dataclasses.py b/WebCrawler/venv/Lib/site-packages/astroid/brain/brain_dataclasses.py new file mode 100644 index 0000000..7a25e0c --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/astroid/brain/brain_dataclasses.py @@ -0,0 +1,50 @@ +# Licensed under the LGPL: https://www.gnu.org/licenses/old-licenses/lgpl-2.1.en.html +# For details: https://github.com/PyCQA/astroid/blob/master/COPYING.LESSER +""" +Astroid hook for the dataclasses library +""" + +import astroid +from astroid import MANAGER + + +DATACLASSES_DECORATORS = frozenset(("dataclasses.dataclass", "dataclass")) + + +def is_decorated_with_dataclass(node, decorator_names=DATACLASSES_DECORATORS): + """Return True if a decorated node has a `dataclass` decorator applied.""" + if not node.decorators: + return False + for decorator_attribute in node.decorators.nodes: + if isinstance(decorator_attribute, astroid.Call): # decorator with arguments + decorator_attribute = decorator_attribute.func + if decorator_attribute.as_string() in decorator_names: + return True + return False + + +def dataclass_transform(node): + """Rewrite a dataclass to be easily understood by pylint""" + + for assign_node in node.body: + if not isinstance(assign_node, (astroid.AnnAssign, astroid.Assign)): + continue + + targets = ( + assign_node.targets + if hasattr(assign_node, "targets") + else [assign_node.target] + ) + for target in targets: + rhs_node = astroid.Unknown( + lineno=assign_node.lineno, + col_offset=assign_node.col_offset, + parent=assign_node, + ) + node.instance_attrs[target.name] = [rhs_node] + node.locals[target.name] = [rhs_node] + + +MANAGER.register_transform( + astroid.ClassDef, dataclass_transform, is_decorated_with_dataclass +) diff --git a/WebCrawler/venv/Lib/site-packages/astroid/brain/brain_dateutil.py b/WebCrawler/venv/Lib/site-packages/astroid/brain/brain_dateutil.py new file mode 100644 index 0000000..9fdb9fd --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/astroid/brain/brain_dateutil.py @@ -0,0 +1,28 @@ +# Copyright (c) 2015-2016, 2018 Claudiu Popa +# Copyright (c) 2015 raylu +# Copyright (c) 2016 Ceridwen + +# Licensed under the LGPL: https://www.gnu.org/licenses/old-licenses/lgpl-2.1.en.html +# For details: https://github.com/PyCQA/astroid/blob/master/COPYING.LESSER + +"""Astroid hooks for dateutil""" + +import textwrap + +from astroid import MANAGER, register_module_extender +from astroid.builder import AstroidBuilder + + +def dateutil_transform(): + return AstroidBuilder(MANAGER).string_build( + textwrap.dedent( + """ + import datetime + def parse(timestr, parserinfo=None, **kwargs): + return datetime.datetime() + """ + ) + ) + + +register_module_extender(MANAGER, "dateutil.parser", dateutil_transform) diff --git a/WebCrawler/venv/Lib/site-packages/astroid/brain/brain_fstrings.py b/WebCrawler/venv/Lib/site-packages/astroid/brain/brain_fstrings.py new file mode 100644 index 0000000..298d58a --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/astroid/brain/brain_fstrings.py @@ -0,0 +1,51 @@ +# Copyright (c) 2017-2018 Claudiu Popa + +# Licensed under the LGPL: https://www.gnu.org/licenses/old-licenses/lgpl-2.1.en.html +# For details: https://github.com/PyCQA/astroid/blob/master/COPYING.LESSER +import collections +import sys + +import astroid + + +def _clone_node_with_lineno(node, parent, lineno): + cls = node.__class__ + other_fields = node._other_fields + _astroid_fields = node._astroid_fields + init_params = {"lineno": lineno, "col_offset": node.col_offset, "parent": parent} + postinit_params = {param: getattr(node, param) for param in _astroid_fields} + if other_fields: + init_params.update({param: getattr(node, param) for param in other_fields}) + new_node = cls(**init_params) + if hasattr(node, "postinit") and _astroid_fields: + for param, child in postinit_params.items(): + if child and not isinstance(child, collections.Sequence): + cloned_child = _clone_node_with_lineno( + node=child, lineno=new_node.lineno, parent=new_node + ) + postinit_params[param] = cloned_child + new_node.postinit(**postinit_params) + return new_node + + +def _transform_formatted_value(node): + if node.value and node.value.lineno == 1: + if node.lineno != node.value.lineno: + new_node = astroid.FormattedValue( + lineno=node.lineno, col_offset=node.col_offset, parent=node.parent + ) + new_value = _clone_node_with_lineno( + node=node.value, lineno=node.lineno, parent=new_node + ) + new_node.postinit(value=new_value, format_spec=node.format_spec) + return new_node + + +if sys.version_info[:2] >= (3, 6): + # TODO: this fix tries to *patch* http://bugs.python.org/issue29051 + # The problem is that FormattedValue.value, which is a Name node, + # has wrong line numbers, usually 1. This creates problems for pylint, + # which expects correct line numbers for things such as message control. + astroid.MANAGER.register_transform( + astroid.FormattedValue, _transform_formatted_value + ) diff --git a/WebCrawler/venv/Lib/site-packages/astroid/brain/brain_functools.py b/WebCrawler/venv/Lib/site-packages/astroid/brain/brain_functools.py new file mode 100644 index 0000000..d6c6069 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/astroid/brain/brain_functools.py @@ -0,0 +1,159 @@ +# Copyright (c) 2016, 2018-2020 Claudiu Popa +# Copyright (c) 2018 hippo91 +# Copyright (c) 2018 Bryce Guinta + +"""Astroid hooks for understanding functools library module.""" +from functools import partial +from itertools import chain + +import astroid +from astroid import arguments +from astroid import BoundMethod +from astroid import extract_node +from astroid import helpers +from astroid.interpreter import objectmodel +from astroid import MANAGER +from astroid import objects + + +LRU_CACHE = "functools.lru_cache" + + +class LruWrappedModel(objectmodel.FunctionModel): + """Special attribute model for functions decorated with functools.lru_cache. + + The said decorators patches at decoration time some functions onto + the decorated function. + """ + + @property + def attr___wrapped__(self): + return self._instance + + @property + def attr_cache_info(self): + cache_info = extract_node( + """ + from functools import _CacheInfo + _CacheInfo(0, 0, 0, 0) + """ + ) + + class CacheInfoBoundMethod(BoundMethod): + def infer_call_result(self, caller, context=None): + yield helpers.safe_infer(cache_info) + + return CacheInfoBoundMethod(proxy=self._instance, bound=self._instance) + + @property + def attr_cache_clear(self): + node = extract_node("""def cache_clear(self): pass""") + return BoundMethod(proxy=node, bound=self._instance.parent.scope()) + + +def _transform_lru_cache(node, context=None): + # TODO: this is not ideal, since the node should be immutable, + # but due to https://github.com/PyCQA/astroid/issues/354, + # there's not much we can do now. + # Replacing the node would work partially, because, + # in pylint, the old node would still be available, leading + # to spurious false positives. + node.special_attributes = LruWrappedModel()(node) + return + + +def _functools_partial_inference(node, context=None): + call = arguments.CallSite.from_call(node, context=context) + number_of_positional = len(call.positional_arguments) + if number_of_positional < 1: + raise astroid.UseInferenceDefault( + "functools.partial takes at least one argument" + ) + if number_of_positional == 1 and not call.keyword_arguments: + raise astroid.UseInferenceDefault( + "functools.partial needs at least to have some filled arguments" + ) + + partial_function = call.positional_arguments[0] + try: + inferred_wrapped_function = next(partial_function.infer(context=context)) + except astroid.InferenceError as exc: + raise astroid.UseInferenceDefault from exc + if inferred_wrapped_function is astroid.Uninferable: + raise astroid.UseInferenceDefault("Cannot infer the wrapped function") + if not isinstance(inferred_wrapped_function, astroid.FunctionDef): + raise astroid.UseInferenceDefault("The wrapped function is not a function") + + # Determine if the passed keywords into the callsite are supported + # by the wrapped function. + function_parameters = chain( + inferred_wrapped_function.args.args or (), + inferred_wrapped_function.args.posonlyargs or (), + inferred_wrapped_function.args.kwonlyargs or (), + ) + parameter_names = set( + param.name + for param in function_parameters + if isinstance(param, astroid.AssignName) + ) + if set(call.keyword_arguments) - parameter_names: + raise astroid.UseInferenceDefault( + "wrapped function received unknown parameters" + ) + + partial_function = objects.PartialFunction( + call, + name=inferred_wrapped_function.name, + doc=inferred_wrapped_function.doc, + lineno=inferred_wrapped_function.lineno, + col_offset=inferred_wrapped_function.col_offset, + parent=inferred_wrapped_function.parent, + ) + partial_function.postinit( + args=inferred_wrapped_function.args, + body=inferred_wrapped_function.body, + decorators=inferred_wrapped_function.decorators, + returns=inferred_wrapped_function.returns, + type_comment_returns=inferred_wrapped_function.type_comment_returns, + type_comment_args=inferred_wrapped_function.type_comment_args, + ) + return iter((partial_function,)) + + +def _looks_like_lru_cache(node): + """Check if the given function node is decorated with lru_cache.""" + if not node.decorators: + return False + for decorator in node.decorators.nodes: + if not isinstance(decorator, astroid.Call): + continue + if _looks_like_functools_member(decorator, "lru_cache"): + return True + return False + + +def _looks_like_functools_member(node, member): + """Check if the given Call node is a functools.partial call""" + if isinstance(node.func, astroid.Name): + return node.func.name == member + elif isinstance(node.func, astroid.Attribute): + return ( + node.func.attrname == member + and isinstance(node.func.expr, astroid.Name) + and node.func.expr.name == "functools" + ) + + +_looks_like_partial = partial(_looks_like_functools_member, member="partial") + + +MANAGER.register_transform( + astroid.FunctionDef, _transform_lru_cache, _looks_like_lru_cache +) + + +MANAGER.register_transform( + astroid.Call, + astroid.inference_tip(_functools_partial_inference), + _looks_like_partial, +) diff --git a/WebCrawler/venv/Lib/site-packages/astroid/brain/brain_gi.py b/WebCrawler/venv/Lib/site-packages/astroid/brain/brain_gi.py new file mode 100644 index 0000000..e49f3a2 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/astroid/brain/brain_gi.py @@ -0,0 +1,253 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2013-2014 LOGILAB S.A. (Paris, FRANCE) +# Copyright (c) 2014 Google, Inc. +# Copyright (c) 2014 Cole Robinson +# Copyright (c) 2015-2016, 2018 Claudiu Popa +# Copyright (c) 2015-2016 Ceridwen +# Copyright (c) 2015 David Shea +# Copyright (c) 2016 Jakub Wilk +# Copyright (c) 2016 Giuseppe Scrivano +# Copyright (c) 2018 Christoph Reiter +# Copyright (c) 2019 Philipp Hörist + +# Licensed under the LGPL: https://www.gnu.org/licenses/old-licenses/lgpl-2.1.en.html +# For details: https://github.com/PyCQA/astroid/blob/master/COPYING.LESSER + +"""Astroid hooks for the Python 2 GObject introspection bindings. + +Helps with understanding everything imported from 'gi.repository' +""" + +import inspect +import itertools +import sys +import re +import warnings + +from astroid import MANAGER, AstroidBuildingError, nodes +from astroid.builder import AstroidBuilder + + +_inspected_modules = {} + +_identifier_re = r"^[A-Za-z_]\w*$" + +_special_methods = frozenset( + { + "__lt__", + "__le__", + "__eq__", + "__ne__", + "__ge__", + "__gt__", + "__iter__", + "__getitem__", + "__setitem__", + "__delitem__", + "__len__", + "__bool__", + "__nonzero__", + "__next__", + "__str__", + "__len__", + "__contains__", + "__enter__", + "__exit__", + "__repr__", + "__getattr__", + "__setattr__", + "__delattr__", + "__del__", + "__hash__", + } +) + + +def _gi_build_stub(parent): + """ + Inspect the passed module recursively and build stubs for functions, + classes, etc. + """ + classes = {} + functions = {} + constants = {} + methods = {} + for name in dir(parent): + if name.startswith("__") and name not in _special_methods: + continue + + # Check if this is a valid name in python + if not re.match(_identifier_re, name): + continue + + try: + obj = getattr(parent, name) + except: + continue + + if inspect.isclass(obj): + classes[name] = obj + elif inspect.isfunction(obj) or inspect.isbuiltin(obj): + functions[name] = obj + elif inspect.ismethod(obj) or inspect.ismethoddescriptor(obj): + methods[name] = obj + elif ( + str(obj).startswith(", ) + # Only accept function calls with two constant arguments + if len(node.args) != 2: + return False + + if not all(isinstance(arg, nodes.Const) for arg in node.args): + return False + + func = node.func + if isinstance(func, nodes.Attribute): + if func.attrname != "require_version": + return False + if isinstance(func.expr, nodes.Name) and func.expr.name == "gi": + return True + + return False + + if isinstance(func, nodes.Name): + return func.name == "require_version" + + return False + + +def _register_require_version(node): + # Load the gi.require_version locally + try: + import gi + + gi.require_version(node.args[0].value, node.args[1].value) + except Exception: + pass + + return node + + +MANAGER.register_failed_import_hook(_import_gi_module) +MANAGER.register_transform( + nodes.Call, _register_require_version, _looks_like_require_version +) diff --git a/WebCrawler/venv/Lib/site-packages/astroid/brain/brain_hashlib.py b/WebCrawler/venv/Lib/site-packages/astroid/brain/brain_hashlib.py new file mode 100644 index 0000000..eb34e15 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/astroid/brain/brain_hashlib.py @@ -0,0 +1,69 @@ +# Copyright (c) 2016, 2018 Claudiu Popa +# Copyright (c) 2018 David Poirier +# Copyright (c) 2018 wgehalo +# Copyright (c) 2018 Ioana Tagirta + +# Licensed under the LGPL: https://www.gnu.org/licenses/old-licenses/lgpl-2.1.en.html +# For details: https://github.com/PyCQA/astroid/blob/master/COPYING.LESSER +import sys + +import six + +import astroid + +PY36 = sys.version_info >= (3, 6) + + +def _hashlib_transform(): + signature = "value=''" + template = """ + class %(name)s(object): + def __init__(self, %(signature)s): pass + def digest(self): + return %(digest)s + def copy(self): + return self + def update(self, value): pass + def hexdigest(self): + return '' + @property + def name(self): + return %(name)r + @property + def block_size(self): + return 1 + @property + def digest_size(self): + return 1 + """ + algorithms_with_signature = dict.fromkeys( + ["md5", "sha1", "sha224", "sha256", "sha384", "sha512"], signature + ) + if PY36: + blake2b_signature = "data=b'', *, digest_size=64, key=b'', salt=b'', \ + person=b'', fanout=1, depth=1, leaf_size=0, node_offset=0, \ + node_depth=0, inner_size=0, last_node=False" + blake2s_signature = "data=b'', *, digest_size=32, key=b'', salt=b'', \ + person=b'', fanout=1, depth=1, leaf_size=0, node_offset=0, \ + node_depth=0, inner_size=0, last_node=False" + new_algorithms = dict.fromkeys( + ["sha3_224", "sha3_256", "sha3_384", "sha3_512", "shake_128", "shake_256"], + signature, + ) + algorithms_with_signature.update(new_algorithms) + algorithms_with_signature.update( + {"blake2b": blake2b_signature, "blake2s": blake2s_signature} + ) + classes = "".join( + template + % { + "name": hashfunc, + "digest": 'b""' if six.PY3 else '""', + "signature": signature, + } + for hashfunc, signature in algorithms_with_signature.items() + ) + return astroid.parse(classes) + + +astroid.register_module_extender(astroid.MANAGER, "hashlib", _hashlib_transform) diff --git a/WebCrawler/venv/Lib/site-packages/astroid/brain/brain_http.py b/WebCrawler/venv/Lib/site-packages/astroid/brain/brain_http.py new file mode 100644 index 0000000..b16464e --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/astroid/brain/brain_http.py @@ -0,0 +1,211 @@ +# Copyright (c) 2018-2019 Claudiu Popa + +# Licensed under the LGPL: https://www.gnu.org/licenses/old-licenses/lgpl-2.1.en.html +# For details: https://github.com/PyCQA/astroid/blob/master/COPYING.LESSER + +"""Astroid brain hints for some of the `http` module.""" +import textwrap + +import astroid +from astroid.builder import AstroidBuilder + + +def _http_transform(): + code = textwrap.dedent( + """ + from collections import namedtuple + _HTTPStatus = namedtuple('_HTTPStatus', 'value phrase description') + + class HTTPStatus: + + @property + def phrase(self): + return "" + @property + def value(self): + return 0 + @property + def description(self): + return "" + + # informational + CONTINUE = _HTTPStatus(100, 'Continue', 'Request received, please continue') + SWITCHING_PROTOCOLS = _HTTPStatus(101, 'Switching Protocols', + 'Switching to new protocol; obey Upgrade header') + PROCESSING = _HTTPStatus(102, 'Processing', '') + OK = _HTTPStatus(200, 'OK', 'Request fulfilled, document follows') + CREATED = _HTTPStatus(201, 'Created', 'Document created, URL follows') + ACCEPTED = _HTTPStatus(202, 'Accepted', + 'Request accepted, processing continues off-line') + NON_AUTHORITATIVE_INFORMATION = _HTTPStatus(203, + 'Non-Authoritative Information', 'Request fulfilled from cache') + NO_CONTENT = _HTTPStatus(204, 'No Content', 'Request fulfilled, nothing follows') + RESET_CONTENT =_HTTPStatus(205, 'Reset Content', 'Clear input form for further input') + PARTIAL_CONTENT = _HTTPStatus(206, 'Partial Content', 'Partial content follows') + MULTI_STATUS = _HTTPStatus(207, 'Multi-Status', '') + ALREADY_REPORTED = _HTTPStatus(208, 'Already Reported', '') + IM_USED = _HTTPStatus(226, 'IM Used', '') + MULTIPLE_CHOICES = _HTTPStatus(300, 'Multiple Choices', + 'Object has several resources -- see URI list') + MOVED_PERMANENTLY = _HTTPStatus(301, 'Moved Permanently', + 'Object moved permanently -- see URI list') + FOUND = _HTTPStatus(302, 'Found', 'Object moved temporarily -- see URI list') + SEE_OTHER = _HTTPStatus(303, 'See Other', 'Object moved -- see Method and URL list') + NOT_MODIFIED = _HTTPStatus(304, 'Not Modified', + 'Document has not changed since given time') + USE_PROXY = _HTTPStatus(305, 'Use Proxy', + 'You must use proxy specified in Location to access this resource') + TEMPORARY_REDIRECT = _HTTPStatus(307, 'Temporary Redirect', + 'Object moved temporarily -- see URI list') + PERMANENT_REDIRECT = _HTTPStatus(308, 'Permanent Redirect', + 'Object moved permanently -- see URI list') + BAD_REQUEST = _HTTPStatus(400, 'Bad Request', + 'Bad request syntax or unsupported method') + UNAUTHORIZED = _HTTPStatus(401, 'Unauthorized', + 'No permission -- see authorization schemes') + PAYMENT_REQUIRED = _HTTPStatus(402, 'Payment Required', + 'No payment -- see charging schemes') + FORBIDDEN = _HTTPStatus(403, 'Forbidden', + 'Request forbidden -- authorization will not help') + NOT_FOUND = _HTTPStatus(404, 'Not Found', + 'Nothing matches the given URI') + METHOD_NOT_ALLOWED = _HTTPStatus(405, 'Method Not Allowed', + 'Specified method is invalid for this resource') + NOT_ACCEPTABLE = _HTTPStatus(406, 'Not Acceptable', + 'URI not available in preferred format') + PROXY_AUTHENTICATION_REQUIRED = _HTTPStatus(407, + 'Proxy Authentication Required', + 'You must authenticate with this proxy before proceeding') + REQUEST_TIMEOUT = _HTTPStatus(408, 'Request Timeout', + 'Request timed out; try again later') + CONFLICT = _HTTPStatus(409, 'Conflict', 'Request conflict') + GONE = _HTTPStatus(410, 'Gone', + 'URI no longer exists and has been permanently removed') + LENGTH_REQUIRED = _HTTPStatus(411, 'Length Required', + 'Client must specify Content-Length') + PRECONDITION_FAILED = _HTTPStatus(412, 'Precondition Failed', + 'Precondition in headers is false') + REQUEST_ENTITY_TOO_LARGE = _HTTPStatus(413, 'Request Entity Too Large', + 'Entity is too large') + REQUEST_URI_TOO_LONG = _HTTPStatus(414, 'Request-URI Too Long', + 'URI is too long') + UNSUPPORTED_MEDIA_TYPE = _HTTPStatus(415, 'Unsupported Media Type', + 'Entity body in unsupported format') + REQUESTED_RANGE_NOT_SATISFIABLE = _HTTPStatus(416, + 'Requested Range Not Satisfiable', + 'Cannot satisfy request range') + EXPECTATION_FAILED = _HTTPStatus(417, 'Expectation Failed', + 'Expect condition could not be satisfied') + MISDIRECTED_REQUEST = _HTTPStatus(421, 'Misdirected Request', + 'Server is not able to produce a response') + UNPROCESSABLE_ENTITY = _HTTPStatus(422, 'Unprocessable Entity') + LOCKED = _HTTPStatus(423, 'Locked') + FAILED_DEPENDENCY = _HTTPStatus(424, 'Failed Dependency') + UPGRADE_REQUIRED = _HTTPStatus(426, 'Upgrade Required') + PRECONDITION_REQUIRED = _HTTPStatus(428, 'Precondition Required', + 'The origin server requires the request to be conditional') + TOO_MANY_REQUESTS = _HTTPStatus(429, 'Too Many Requests', + 'The user has sent too many requests in ' + 'a given amount of time ("rate limiting")') + REQUEST_HEADER_FIELDS_TOO_LARGE = _HTTPStatus(431, + 'Request Header Fields Too Large', + 'The server is unwilling to process the request because its header ' + 'fields are too large') + UNAVAILABLE_FOR_LEGAL_REASONS = _HTTPStatus(451, + 'Unavailable For Legal Reasons', + 'The server is denying access to the ' + 'resource as a consequence of a legal demand') + INTERNAL_SERVER_ERROR = _HTTPStatus(500, 'Internal Server Error', + 'Server got itself in trouble') + NOT_IMPLEMENTED = _HTTPStatus(501, 'Not Implemented', + 'Server does not support this operation') + BAD_GATEWAY = _HTTPStatus(502, 'Bad Gateway', + 'Invalid responses from another server/proxy') + SERVICE_UNAVAILABLE = _HTTPStatus(503, 'Service Unavailable', + 'The server cannot process the request due to a high load') + GATEWAY_TIMEOUT = _HTTPStatus(504, 'Gateway Timeout', + 'The gateway server did not receive a timely response') + HTTP_VERSION_NOT_SUPPORTED = _HTTPStatus(505, 'HTTP Version Not Supported', + 'Cannot fulfill request') + VARIANT_ALSO_NEGOTIATES = _HTTPStatus(506, 'Variant Also Negotiates') + INSUFFICIENT_STORAGE = _HTTPStatus(507, 'Insufficient Storage') + LOOP_DETECTED = _HTTPStatus(508, 'Loop Detected') + NOT_EXTENDED = _HTTPStatus(510, 'Not Extended') + NETWORK_AUTHENTICATION_REQUIRED = _HTTPStatus(511, + 'Network Authentication Required', + 'The client needs to authenticate to gain network access') + """ + ) + return AstroidBuilder(astroid.MANAGER).string_build(code) + + +def _http_client_transform(): + return AstroidBuilder(astroid.MANAGER).string_build( + textwrap.dedent( + """ + from http import HTTPStatus + + CONTINUE = HTTPStatus.CONTINUE + SWITCHING_PROTOCOLS = HTTPStatus.SWITCHING_PROTOCOLS + PROCESSING = HTTPStatus.PROCESSING + OK = HTTPStatus.OK + CREATED = HTTPStatus.CREATED + ACCEPTED = HTTPStatus.ACCEPTED + NON_AUTHORITATIVE_INFORMATION = HTTPStatus.NON_AUTHORITATIVE_INFORMATION + NO_CONTENT = HTTPStatus.NO_CONTENT + RESET_CONTENT = HTTPStatus.RESET_CONTENT + PARTIAL_CONTENT = HTTPStatus.PARTIAL_CONTENT + MULTI_STATUS = HTTPStatus.MULTI_STATUS + ALREADY_REPORTED = HTTPStatus.ALREADY_REPORTED + IM_USED = HTTPStatus.IM_USED + MULTIPLE_CHOICES = HTTPStatus.MULTIPLE_CHOICES + MOVED_PERMANENTLY = HTTPStatus.MOVED_PERMANENTLY + FOUND = HTTPStatus.FOUND + SEE_OTHER = HTTPStatus.SEE_OTHER + NOT_MODIFIED = HTTPStatus.NOT_MODIFIED + USE_PROXY = HTTPStatus.USE_PROXY + TEMPORARY_REDIRECT = HTTPStatus.TEMPORARY_REDIRECT + PERMANENT_REDIRECT = HTTPStatus.PERMANENT_REDIRECT + BAD_REQUEST = HTTPStatus.BAD_REQUEST + UNAUTHORIZED = HTTPStatus.UNAUTHORIZED + PAYMENT_REQUIRED = HTTPStatus.PAYMENT_REQUIRED + FORBIDDEN = HTTPStatus.FORBIDDEN + NOT_FOUND = HTTPStatus.NOT_FOUND + METHOD_NOT_ALLOWED = HTTPStatus.METHOD_NOT_ALLOWED + NOT_ACCEPTABLE = HTTPStatus.NOT_ACCEPTABLE + PROXY_AUTHENTICATION_REQUIRED = HTTPStatus.PROXY_AUTHENTICATION_REQUIRED + REQUEST_TIMEOUT = HTTPStatus.REQUEST_TIMEOUT + CONFLICT = HTTPStatus.CONFLICT + GONE = HTTPStatus.GONE + LENGTH_REQUIRED = HTTPStatus.LENGTH_REQUIRED + PRECONDITION_FAILED = HTTPStatus.PRECONDITION_FAILED + REQUEST_ENTITY_TOO_LARGE = HTTPStatus.REQUEST_ENTITY_TOO_LARGE + REQUEST_URI_TOO_LONG = HTTPStatus.REQUEST_URI_TOO_LONG + UNSUPPORTED_MEDIA_TYPE = HTTPStatus.UNSUPPORTED_MEDIA_TYPE + REQUESTED_RANGE_NOT_SATISFIABLE = HTTPStatus.REQUESTED_RANGE_NOT_SATISFIABLE + EXPECTATION_FAILED = HTTPStatus.EXPECTATION_FAILED + UNPROCESSABLE_ENTITY = HTTPStatus.UNPROCESSABLE_ENTITY + LOCKED = HTTPStatus.LOCKED + FAILED_DEPENDENCY = HTTPStatus.FAILED_DEPENDENCY + UPGRADE_REQUIRED = HTTPStatus.UPGRADE_REQUIRED + PRECONDITION_REQUIRED = HTTPStatus.PRECONDITION_REQUIRED + TOO_MANY_REQUESTS = HTTPStatus.TOO_MANY_REQUESTS + REQUEST_HEADER_FIELDS_TOO_LARGE = HTTPStatus.REQUEST_HEADER_FIELDS_TOO_LARGE + INTERNAL_SERVER_ERROR = HTTPStatus.INTERNAL_SERVER_ERROR + NOT_IMPLEMENTED = HTTPStatus.NOT_IMPLEMENTED + BAD_GATEWAY = HTTPStatus.BAD_GATEWAY + SERVICE_UNAVAILABLE = HTTPStatus.SERVICE_UNAVAILABLE + GATEWAY_TIMEOUT = HTTPStatus.GATEWAY_TIMEOUT + HTTP_VERSION_NOT_SUPPORTED = HTTPStatus.HTTP_VERSION_NOT_SUPPORTED + VARIANT_ALSO_NEGOTIATES = HTTPStatus.VARIANT_ALSO_NEGOTIATES + INSUFFICIENT_STORAGE = HTTPStatus.INSUFFICIENT_STORAGE + LOOP_DETECTED = HTTPStatus.LOOP_DETECTED + NOT_EXTENDED = HTTPStatus.NOT_EXTENDED + NETWORK_AUTHENTICATION_REQUIRED = HTTPStatus.NETWORK_AUTHENTICATION_REQUIRED + """ + ) + ) + + +astroid.register_module_extender(astroid.MANAGER, "http", _http_transform) +astroid.register_module_extender(astroid.MANAGER, "http.client", _http_client_transform) diff --git a/WebCrawler/venv/Lib/site-packages/astroid/brain/brain_io.py b/WebCrawler/venv/Lib/site-packages/astroid/brain/brain_io.py new file mode 100644 index 0000000..c745311 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/astroid/brain/brain_io.py @@ -0,0 +1,45 @@ +# Copyright (c) 2016, 2018 Claudiu Popa + +# Licensed under the LGPL: https://www.gnu.org/licenses/old-licenses/lgpl-2.1.en.html +# For details: https://github.com/PyCQA/astroid/blob/master/COPYING.LESSER + +"""Astroid brain hints for some of the _io C objects.""" + +import astroid + + +BUFFERED = {"BufferedWriter", "BufferedReader"} +TextIOWrapper = "TextIOWrapper" +FileIO = "FileIO" +BufferedWriter = "BufferedWriter" + + +def _generic_io_transform(node, name, cls): + """Transform the given name, by adding the given *class* as a member of the node.""" + + io_module = astroid.MANAGER.ast_from_module_name("_io") + attribute_object = io_module[cls] + instance = attribute_object.instantiate_class() + node.locals[name] = [instance] + + +def _transform_text_io_wrapper(node): + # This is not always correct, since it can vary with the type of the descriptor, + # being stdout, stderr or stdin. But we cannot get access to the name of the + # stream, which is why we are using the BufferedWriter class as a default + # value + return _generic_io_transform(node, name="buffer", cls=BufferedWriter) + + +def _transform_buffered(node): + return _generic_io_transform(node, name="raw", cls=FileIO) + + +astroid.MANAGER.register_transform( + astroid.ClassDef, _transform_buffered, lambda node: node.name in BUFFERED +) +astroid.MANAGER.register_transform( + astroid.ClassDef, + _transform_text_io_wrapper, + lambda node: node.name == TextIOWrapper, +) diff --git a/WebCrawler/venv/Lib/site-packages/astroid/brain/brain_mechanize.py b/WebCrawler/venv/Lib/site-packages/astroid/brain/brain_mechanize.py new file mode 100644 index 0000000..ef62c53 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/astroid/brain/brain_mechanize.py @@ -0,0 +1,29 @@ +# Copyright (c) 2012-2013 LOGILAB S.A. (Paris, FRANCE) +# Copyright (c) 2014 Google, Inc. +# Copyright (c) 2015-2016, 2018 Claudiu Popa +# Copyright (c) 2016 Ceridwen + +# Licensed under the LGPL: https://www.gnu.org/licenses/old-licenses/lgpl-2.1.en.html +# For details: https://github.com/PyCQA/astroid/blob/master/COPYING.LESSER + +from astroid import MANAGER, register_module_extender +from astroid.builder import AstroidBuilder + + +def mechanize_transform(): + return AstroidBuilder(MANAGER).string_build( + """ + +class Browser(object): + def open(self, url, data=None, timeout=None): + return None + def open_novisit(self, url, data=None, timeout=None): + return None + def open_local_file(self, filename): + return None + +""" + ) + + +register_module_extender(MANAGER, "mechanize", mechanize_transform) diff --git a/WebCrawler/venv/Lib/site-packages/astroid/brain/brain_multiprocessing.py b/WebCrawler/venv/Lib/site-packages/astroid/brain/brain_multiprocessing.py new file mode 100644 index 0000000..3629b03 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/astroid/brain/brain_multiprocessing.py @@ -0,0 +1,107 @@ +# Copyright (c) 2016, 2018 Claudiu Popa +# Copyright (c) 2019 Hugo van Kemenade + +# Licensed under the LGPL: https://www.gnu.org/licenses/old-licenses/lgpl-2.1.en.html +# For details: https://github.com/PyCQA/astroid/blob/master/COPYING.LESSER + +import sys + +import astroid +from astroid import exceptions + + +def _multiprocessing_transform(): + module = astroid.parse( + """ + from multiprocessing.managers import SyncManager + def Manager(): + return SyncManager() + """ + ) + # Multiprocessing uses a getattr lookup inside contexts, + # in order to get the attributes they need. Since it's extremely + # dynamic, we use this approach to fake it. + node = astroid.parse( + """ + from multiprocessing.context import DefaultContext, BaseContext + default = DefaultContext() + base = BaseContext() + """ + ) + try: + context = next(node["default"].infer()) + base = next(node["base"].infer()) + except exceptions.InferenceError: + return module + + for node in (context, base): + for key, value in node.locals.items(): + if key.startswith("_"): + continue + + value = value[0] + if isinstance(value, astroid.FunctionDef): + # We need to rebound this, since otherwise + # it will have an extra argument (self). + value = astroid.BoundMethod(value, node) + module[key] = value + return module + + +def _multiprocessing_managers_transform(): + return astroid.parse( + """ + import array + import threading + import multiprocessing.pool as pool + + import six + + class Namespace(object): + pass + + class Value(object): + def __init__(self, typecode, value, lock=True): + self._typecode = typecode + self._value = value + def get(self): + return self._value + def set(self, value): + self._value = value + def __repr__(self): + return '%s(%r, %r)'%(type(self).__name__, self._typecode, self._value) + value = property(get, set) + + def Array(typecode, sequence, lock=True): + return array.array(typecode, sequence) + + class SyncManager(object): + Queue = JoinableQueue = six.moves.queue.Queue + Event = threading.Event + RLock = threading.RLock + BoundedSemaphore = threading.BoundedSemaphore + Condition = threading.Condition + Barrier = threading.Barrier + Pool = pool.Pool + list = list + dict = dict + Value = Value + Array = Array + Namespace = Namespace + __enter__ = lambda self: self + __exit__ = lambda *args: args + + def start(self, initializer=None, initargs=None): + pass + def shutdown(self): + pass + """ + ) + + +astroid.register_module_extender( + astroid.MANAGER, "multiprocessing.managers", _multiprocessing_managers_transform +) +astroid.register_module_extender( + astroid.MANAGER, "multiprocessing", _multiprocessing_transform +) diff --git a/WebCrawler/venv/Lib/site-packages/astroid/brain/brain_namedtuple_enum.py b/WebCrawler/venv/Lib/site-packages/astroid/brain/brain_namedtuple_enum.py new file mode 100644 index 0000000..13fcf79 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/astroid/brain/brain_namedtuple_enum.py @@ -0,0 +1,455 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2012-2015 LOGILAB S.A. (Paris, FRANCE) +# Copyright (c) 2013-2014 Google, Inc. +# Copyright (c) 2014-2020 Claudiu Popa +# Copyright (c) 2014 Eevee (Alex Munroe) +# Copyright (c) 2015-2016 Ceridwen +# Copyright (c) 2015 Dmitry Pribysh +# Copyright (c) 2015 David Shea +# Copyright (c) 2015 Philip Lorenz +# Copyright (c) 2016 Jakub Wilk +# Copyright (c) 2016 Mateusz Bysiek +# Copyright (c) 2017 Hugo +# Copyright (c) 2017 Łukasz Rogalski +# Copyright (c) 2018 Ville Skyttä +# Copyright (c) 2019 Ashley Whetter + +# Licensed under the LGPL: https://www.gnu.org/licenses/old-licenses/lgpl-2.1.en.html +# For details: https://github.com/PyCQA/astroid/blob/master/COPYING.LESSER + +"""Astroid hooks for the Python standard library.""" + +import functools +import keyword +from textwrap import dedent + +from astroid import MANAGER, UseInferenceDefault, inference_tip, InferenceError +from astroid import arguments +from astroid import exceptions +from astroid import nodes +from astroid.builder import AstroidBuilder, extract_node +from astroid import util + + +TYPING_NAMEDTUPLE_BASENAMES = {"NamedTuple", "typing.NamedTuple"} +ENUM_BASE_NAMES = { + "Enum", + "IntEnum", + "enum.Enum", + "enum.IntEnum", + "IntFlag", + "enum.IntFlag", +} + + +def _infer_first(node, context): + if node is util.Uninferable: + raise UseInferenceDefault + try: + value = next(node.infer(context=context)) + if value is util.Uninferable: + raise UseInferenceDefault() + else: + return value + except StopIteration: + raise InferenceError() + + +def _find_func_form_arguments(node, context): + def _extract_namedtuple_arg_or_keyword(position, key_name=None): + + if len(args) > position: + return _infer_first(args[position], context) + if key_name and key_name in found_keywords: + return _infer_first(found_keywords[key_name], context) + + args = node.args + keywords = node.keywords + found_keywords = ( + {keyword.arg: keyword.value for keyword in keywords} if keywords else {} + ) + + name = _extract_namedtuple_arg_or_keyword(position=0, key_name="typename") + names = _extract_namedtuple_arg_or_keyword(position=1, key_name="field_names") + if name and names: + return name.value, names + + raise UseInferenceDefault() + + +def infer_func_form(node, base_type, context=None, enum=False): + """Specific inference function for namedtuple or Python 3 enum. """ + # node is a Call node, class name as first argument and generated class + # attributes as second argument + + # namedtuple or enums list of attributes can be a list of strings or a + # whitespace-separate string + try: + name, names = _find_func_form_arguments(node, context) + try: + attributes = names.value.replace(",", " ").split() + except AttributeError: + if not enum: + attributes = [ + _infer_first(const, context).value for const in names.elts + ] + else: + # Enums supports either iterator of (name, value) pairs + # or mappings. + if hasattr(names, "items") and isinstance(names.items, list): + attributes = [ + _infer_first(const[0], context).value + for const in names.items + if isinstance(const[0], nodes.Const) + ] + elif hasattr(names, "elts"): + # Enums can support either ["a", "b", "c"] + # or [("a", 1), ("b", 2), ...], but they can't + # be mixed. + if all(isinstance(const, nodes.Tuple) for const in names.elts): + attributes = [ + _infer_first(const.elts[0], context).value + for const in names.elts + if isinstance(const, nodes.Tuple) + ] + else: + attributes = [ + _infer_first(const, context).value for const in names.elts + ] + else: + raise AttributeError + if not attributes: + raise AttributeError + except (AttributeError, exceptions.InferenceError): + raise UseInferenceDefault() + + attributes = [attr for attr in attributes if " " not in attr] + + # If we can't infer the name of the class, don't crash, up to this point + # we know it is a namedtuple anyway. + name = name or "Uninferable" + # we want to return a Class node instance with proper attributes set + class_node = nodes.ClassDef(name, "docstring") + class_node.parent = node.parent + # set base class=tuple + class_node.bases.append(base_type) + # XXX add __init__(*attributes) method + for attr in attributes: + fake_node = nodes.EmptyNode() + fake_node.parent = class_node + fake_node.attrname = attr + class_node.instance_attrs[attr] = [fake_node] + return class_node, name, attributes + + +def _has_namedtuple_base(node): + """Predicate for class inference tip + + :type node: ClassDef + :rtype: bool + """ + return set(node.basenames) & TYPING_NAMEDTUPLE_BASENAMES + + +def _looks_like(node, name): + func = node.func + if isinstance(func, nodes.Attribute): + return func.attrname == name + if isinstance(func, nodes.Name): + return func.name == name + return False + + +_looks_like_namedtuple = functools.partial(_looks_like, name="namedtuple") +_looks_like_enum = functools.partial(_looks_like, name="Enum") +_looks_like_typing_namedtuple = functools.partial(_looks_like, name="NamedTuple") + + +def infer_named_tuple(node, context=None): + """Specific inference function for namedtuple Call node""" + tuple_base_name = nodes.Name(name="tuple", parent=node.root()) + class_node, name, attributes = infer_func_form( + node, tuple_base_name, context=context + ) + call_site = arguments.CallSite.from_call(node, context=context) + func = next(extract_node("import collections; collections.namedtuple").infer()) + try: + rename = next(call_site.infer_argument(func, "rename", context)).bool_value() + except InferenceError: + rename = False + + if rename: + attributes = _get_renamed_namedtuple_attributes(attributes) + + replace_args = ", ".join("{arg}=None".format(arg=arg) for arg in attributes) + field_def = ( + " {name} = property(lambda self: self[{index:d}], " + "doc='Alias for field number {index:d}')" + ) + field_defs = "\n".join( + field_def.format(name=name, index=index) + for index, name in enumerate(attributes) + ) + fake = AstroidBuilder(MANAGER).string_build( + """ +class %(name)s(tuple): + __slots__ = () + _fields = %(fields)r + def _asdict(self): + return self.__dict__ + @classmethod + def _make(cls, iterable, new=tuple.__new__, len=len): + return new(cls, iterable) + def _replace(self, %(replace_args)s): + return self + def __getnewargs__(self): + return tuple(self) +%(field_defs)s + """ + % { + "name": name, + "fields": attributes, + "field_defs": field_defs, + "replace_args": replace_args, + } + ) + class_node.locals["_asdict"] = fake.body[0].locals["_asdict"] + class_node.locals["_make"] = fake.body[0].locals["_make"] + class_node.locals["_replace"] = fake.body[0].locals["_replace"] + class_node.locals["_fields"] = fake.body[0].locals["_fields"] + for attr in attributes: + class_node.locals[attr] = fake.body[0].locals[attr] + # we use UseInferenceDefault, we can't be a generator so return an iterator + return iter([class_node]) + + +def _get_renamed_namedtuple_attributes(field_names): + names = list(field_names) + seen = set() + for i, name in enumerate(field_names): + if ( + not all(c.isalnum() or c == "_" for c in name) + or keyword.iskeyword(name) + or not name + or name[0].isdigit() + or name.startswith("_") + or name in seen + ): + names[i] = "_%d" % i + seen.add(name) + return tuple(names) + + +def infer_enum(node, context=None): + """ Specific inference function for enum Call node. """ + enum_meta = extract_node( + """ + class EnumMeta(object): + 'docstring' + def __call__(self, node): + class EnumAttribute(object): + name = '' + value = 0 + return EnumAttribute() + def __iter__(self): + class EnumAttribute(object): + name = '' + value = 0 + return [EnumAttribute()] + def __reversed__(self): + class EnumAttribute(object): + name = '' + value = 0 + return (EnumAttribute, ) + def __next__(self): + return next(iter(self)) + def __getitem__(self, attr): + class Value(object): + @property + def name(self): + return '' + @property + def value(self): + return attr + + return Value() + __members__ = [''] + """ + ) + class_node = infer_func_form(node, enum_meta, context=context, enum=True)[0] + return iter([class_node.instantiate_class()]) + + +INT_FLAG_ADDITION_METHODS = """ + def __or__(self, other): + return {name}(self.value | other.value) + def __and__(self, other): + return {name}(self.value & other.value) + def __xor__(self, other): + return {name}(self.value ^ other.value) + def __add__(self, other): + return {name}(self.value + other.value) + def __div__(self, other): + return {name}(self.value / other.value) + def __invert__(self): + return {name}(~self.value) + def __mul__(self, other): + return {name}(self.value * other.value) +""" + + +def infer_enum_class(node): + """ Specific inference for enums. """ + for basename in node.basenames: + # TODO: doesn't handle subclasses yet. This implementation + # is a hack to support enums. + if basename not in ENUM_BASE_NAMES: + continue + if node.root().name == "enum": + # Skip if the class is directly from enum module. + break + for local, values in node.locals.items(): + if any(not isinstance(value, nodes.AssignName) for value in values): + continue + + targets = [] + stmt = values[0].statement() + if isinstance(stmt, nodes.Assign): + if isinstance(stmt.targets[0], nodes.Tuple): + targets = stmt.targets[0].itered() + else: + targets = stmt.targets + elif isinstance(stmt, nodes.AnnAssign): + targets = [stmt.target] + else: + continue + + inferred_return_value = None + if isinstance(stmt, nodes.Assign): + if isinstance(stmt.value, nodes.Const): + if isinstance(stmt.value.value, str): + inferred_return_value = repr(stmt.value.value) + else: + inferred_return_value = stmt.value.value + else: + inferred_return_value = stmt.value.as_string() + + new_targets = [] + for target in targets: + # Replace all the assignments with our mocked class. + classdef = dedent( + """ + class {name}({types}): + @property + def value(self): + return {return_value} + @property + def name(self): + return "{name}" + """.format( + name=target.name, + types=", ".join(node.basenames), + return_value=inferred_return_value, + ) + ) + if "IntFlag" in basename: + # Alright, we need to add some additional methods. + # Unfortunately we still can't infer the resulting objects as + # Enum members, but once we'll be able to do that, the following + # should result in some nice symbolic execution + classdef += INT_FLAG_ADDITION_METHODS.format(name=target.name) + + fake = AstroidBuilder(MANAGER).string_build(classdef)[target.name] + fake.parent = target.parent + for method in node.mymethods(): + fake.locals[method.name] = [method] + new_targets.append(fake.instantiate_class()) + node.locals[local] = new_targets + break + return node + + +def infer_typing_namedtuple_class(class_node, context=None): + """Infer a subclass of typing.NamedTuple""" + # Check if it has the corresponding bases + annassigns_fields = [ + annassign.target.name + for annassign in class_node.body + if isinstance(annassign, nodes.AnnAssign) + ] + code = dedent( + """ + from collections import namedtuple + namedtuple({typename!r}, {fields!r}) + """ + ).format(typename=class_node.name, fields=",".join(annassigns_fields)) + node = extract_node(code) + generated_class_node = next(infer_named_tuple(node, context)) + for method in class_node.mymethods(): + generated_class_node.locals[method.name] = [method] + + for assign in class_node.body: + if not isinstance(assign, nodes.Assign): + continue + + for target in assign.targets: + attr = target.name + generated_class_node.locals[attr] = class_node.locals[attr] + + return iter((generated_class_node,)) + + +def infer_typing_namedtuple(node, context=None): + """Infer a typing.NamedTuple(...) call.""" + # This is essentially a namedtuple with different arguments + # so we extract the args and infer a named tuple. + try: + func = next(node.func.infer()) + except InferenceError: + raise UseInferenceDefault + + if func.qname() != "typing.NamedTuple": + raise UseInferenceDefault + + if len(node.args) != 2: + raise UseInferenceDefault + + if not isinstance(node.args[1], (nodes.List, nodes.Tuple)): + raise UseInferenceDefault + + names = [] + for elt in node.args[1].elts: + if not isinstance(elt, (nodes.List, nodes.Tuple)): + raise UseInferenceDefault + if len(elt.elts) != 2: + raise UseInferenceDefault + names.append(elt.elts[0].as_string()) + + typename = node.args[0].as_string() + if names: + field_names = "({},)".format(",".join(names)) + else: + field_names = "''" + node = extract_node( + "namedtuple({typename}, {fields})".format(typename=typename, fields=field_names) + ) + return infer_named_tuple(node, context) + + +MANAGER.register_transform( + nodes.Call, inference_tip(infer_named_tuple), _looks_like_namedtuple +) +MANAGER.register_transform(nodes.Call, inference_tip(infer_enum), _looks_like_enum) +MANAGER.register_transform( + nodes.ClassDef, + infer_enum_class, + predicate=lambda cls: any( + basename for basename in cls.basenames if basename in ENUM_BASE_NAMES + ), +) +MANAGER.register_transform( + nodes.ClassDef, inference_tip(infer_typing_namedtuple_class), _has_namedtuple_base +) +MANAGER.register_transform( + nodes.Call, inference_tip(infer_typing_namedtuple), _looks_like_typing_namedtuple +) diff --git a/WebCrawler/venv/Lib/site-packages/astroid/brain/brain_nose.py b/WebCrawler/venv/Lib/site-packages/astroid/brain/brain_nose.py new file mode 100644 index 0000000..d0280a3 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/astroid/brain/brain_nose.py @@ -0,0 +1,77 @@ +# Copyright (c) 2015-2016, 2018 Claudiu Popa +# Copyright (c) 2016 Ceridwen + +# Licensed under the LGPL: https://www.gnu.org/licenses/old-licenses/lgpl-2.1.en.html +# For details: https://github.com/PyCQA/astroid/blob/master/COPYING.LESSER + + +"""Hooks for nose library.""" + +import re +import textwrap + +import astroid +import astroid.builder + +_BUILDER = astroid.builder.AstroidBuilder(astroid.MANAGER) + + +def _pep8(name, caps=re.compile("([A-Z])")): + return caps.sub(lambda m: "_" + m.groups()[0].lower(), name) + + +def _nose_tools_functions(): + """Get an iterator of names and bound methods.""" + module = _BUILDER.string_build( + textwrap.dedent( + """ + import unittest + + class Test(unittest.TestCase): + pass + a = Test() + """ + ) + ) + try: + case = next(module["a"].infer()) + except astroid.InferenceError: + return + for method in case.methods(): + if method.name.startswith("assert") and "_" not in method.name: + pep8_name = _pep8(method.name) + yield pep8_name, astroid.BoundMethod(method, case) + if method.name == "assertEqual": + # nose also exports assert_equals. + yield "assert_equals", astroid.BoundMethod(method, case) + + +def _nose_tools_transform(node): + for method_name, method in _nose_tools_functions(): + node.locals[method_name] = [method] + + +def _nose_tools_trivial_transform(): + """Custom transform for the nose.tools module.""" + stub = _BUILDER.string_build("""__all__ = []""") + all_entries = ["ok_", "eq_"] + + for pep8_name, method in _nose_tools_functions(): + all_entries.append(pep8_name) + stub[pep8_name] = method + + # Update the __all__ variable, since nose.tools + # does this manually with .append. + all_assign = stub["__all__"].parent + all_object = astroid.List(all_entries) + all_object.parent = all_assign + all_assign.value = all_object + return stub + + +astroid.register_module_extender( + astroid.MANAGER, "nose.tools.trivial", _nose_tools_trivial_transform +) +astroid.MANAGER.register_transform( + astroid.Module, _nose_tools_transform, lambda n: n.name == "nose.tools" +) diff --git a/WebCrawler/venv/Lib/site-packages/astroid/brain/brain_numpy_core_fromnumeric.py b/WebCrawler/venv/Lib/site-packages/astroid/brain/brain_numpy_core_fromnumeric.py new file mode 100644 index 0000000..62dfe99 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/astroid/brain/brain_numpy_core_fromnumeric.py @@ -0,0 +1,23 @@ +# Copyright (c) 2019 hippo91 + +# Licensed under the LGPL: https://www.gnu.org/licenses/old-licenses/lgpl-2.1.en.html +# For details: https://github.com/PyCQA/astroid/blob/master/COPYING.LESSER + + +"""Astroid hooks for numpy.core.fromnumeric module.""" + +import astroid + + +def numpy_core_fromnumeric_transform(): + return astroid.parse( + """ + def sum(a, axis=None, dtype=None, out=None, keepdims=None, initial=None): + return numpy.ndarray([0, 0]) + """ + ) + + +astroid.register_module_extender( + astroid.MANAGER, "numpy.core.fromnumeric", numpy_core_fromnumeric_transform +) diff --git a/WebCrawler/venv/Lib/site-packages/astroid/brain/brain_numpy_core_function_base.py b/WebCrawler/venv/Lib/site-packages/astroid/brain/brain_numpy_core_function_base.py new file mode 100644 index 0000000..58aa0a9 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/astroid/brain/brain_numpy_core_function_base.py @@ -0,0 +1,29 @@ +# Copyright (c) 2019 hippo91 + +# Licensed under the LGPL: https://www.gnu.org/licenses/old-licenses/lgpl-2.1.en.html +# For details: https://github.com/PyCQA/astroid/blob/master/COPYING.LESSER + + +"""Astroid hooks for numpy.core.function_base module.""" + +import functools +import astroid +from brain_numpy_utils import looks_like_numpy_member, infer_numpy_member + + +METHODS_TO_BE_INFERRED = { + "linspace": """def linspace(start, stop, num=50, endpoint=True, retstep=False, dtype=None, axis=0): + return numpy.ndarray([0, 0])""", + "logspace": """def logspace(start, stop, num=50, endpoint=True, base=10.0, dtype=None, axis=0): + return numpy.ndarray([0, 0])""", + "geomspace": """def geomspace(start, stop, num=50, endpoint=True, dtype=None, axis=0): + return numpy.ndarray([0, 0])""", +} + +for func_name, func_src in METHODS_TO_BE_INFERRED.items(): + inference_function = functools.partial(infer_numpy_member, func_src) + astroid.MANAGER.register_transform( + astroid.Attribute, + astroid.inference_tip(inference_function), + functools.partial(looks_like_numpy_member, func_name), + ) diff --git a/WebCrawler/venv/Lib/site-packages/astroid/brain/brain_numpy_core_multiarray.py b/WebCrawler/venv/Lib/site-packages/astroid/brain/brain_numpy_core_multiarray.py new file mode 100644 index 0000000..b2e32bc --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/astroid/brain/brain_numpy_core_multiarray.py @@ -0,0 +1,92 @@ +# Copyright (c) 2019-2020 hippo91 + +# Licensed under the LGPL: https://www.gnu.org/licenses/old-licenses/lgpl-2.1.en.html +# For details: https://github.com/PyCQA/astroid/blob/master/COPYING.LESSER + + +"""Astroid hooks for numpy.core.multiarray module.""" + +import functools +import astroid +from brain_numpy_utils import looks_like_numpy_member, infer_numpy_member + + +def numpy_core_multiarray_transform(): + return astroid.parse( + """ + # different functions defined in multiarray.py + def inner(a, b): + return numpy.ndarray([0, 0]) + + def vdot(a, b): + return numpy.ndarray([0, 0]) + """ + ) + + +astroid.register_module_extender( + astroid.MANAGER, "numpy.core.multiarray", numpy_core_multiarray_transform +) + + +METHODS_TO_BE_INFERRED = { + "array": """def array(object, dtype=None, copy=True, order='K', subok=False, ndmin=0): + return numpy.ndarray([0, 0])""", + "dot": """def dot(a, b, out=None): + return numpy.ndarray([0, 0])""", + "empty_like": """def empty_like(a, dtype=None, order='K', subok=True): + return numpy.ndarray((0, 0))""", + "concatenate": """def concatenate(arrays, axis=None, out=None): + return numpy.ndarray((0, 0))""", + "where": """def where(condition, x=None, y=None): + return numpy.ndarray([0, 0])""", + "empty": """def empty(shape, dtype=float, order='C'): + return numpy.ndarray([0, 0])""", + "bincount": """def bincount(x, weights=None, minlength=0): + return numpy.ndarray([0, 0])""", + "busday_count": """def busday_count(begindates, enddates, weekmask='1111100', holidays=[], busdaycal=None, out=None): + return numpy.ndarray([0, 0])""", + "busday_offset": """def busday_offset(dates, offsets, roll='raise', weekmask='1111100', holidays=None, busdaycal=None, out=None): + return numpy.ndarray([0, 0])""", + "can_cast": """def can_cast(from_, to, casting='safe'): + return True""", + "copyto": """def copyto(dst, src, casting='same_kind', where=True): + return None""", + "datetime_as_string": """def datetime_as_string(arr, unit=None, timezone='naive', casting='same_kind'): + return numpy.ndarray([0, 0])""", + "is_busday": """def is_busday(dates, weekmask='1111100', holidays=None, busdaycal=None, out=None): + return numpy.ndarray([0, 0])""", + "lexsort": """def lexsort(keys, axis=-1): + return numpy.ndarray([0, 0])""", + "may_share_memory": """def may_share_memory(a, b, max_work=None): + return True""", + # Not yet available because dtype is not yet present in those brains + # "min_scalar_type": """def min_scalar_type(a): + # return numpy.dtype('int16')""", + "packbits": """def packbits(a, axis=None, bitorder='big'): + return numpy.ndarray([0, 0])""", + # Not yet available because dtype is not yet present in those brains + # "result_type": """def result_type(*arrays_and_dtypes): + # return numpy.dtype('int16')""", + "shares_memory": """def shares_memory(a, b, max_work=None): + return True""", + "unpackbits": """def unpackbits(a, axis=None, count=None, bitorder='big'): + return numpy.ndarray([0, 0])""", + "unravel_index": """def unravel_index(indices, shape, order='C'): + return (numpy.ndarray([0, 0]),)""", + "zeros": """def zeros(shape, dtype=float, order='C'): + return numpy.ndarray([0, 0])""", +} + +for method_name, function_src in METHODS_TO_BE_INFERRED.items(): + inference_function = functools.partial(infer_numpy_member, function_src) + astroid.MANAGER.register_transform( + astroid.Attribute, + astroid.inference_tip(inference_function), + functools.partial(looks_like_numpy_member, method_name), + ) + astroid.MANAGER.register_transform( + astroid.Name, + astroid.inference_tip(inference_function), + functools.partial(looks_like_numpy_member, method_name), + ) diff --git a/WebCrawler/venv/Lib/site-packages/astroid/brain/brain_numpy_core_numeric.py b/WebCrawler/venv/Lib/site-packages/astroid/brain/brain_numpy_core_numeric.py new file mode 100644 index 0000000..2a6f37e --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/astroid/brain/brain_numpy_core_numeric.py @@ -0,0 +1,43 @@ +# Copyright (c) 2019 hippo91 + +# Licensed under the LGPL: https://www.gnu.org/licenses/old-licenses/lgpl-2.1.en.html +# For details: https://github.com/PyCQA/astroid/blob/master/COPYING.LESSER + + +"""Astroid hooks for numpy.core.numeric module.""" + +import functools +import astroid +from brain_numpy_utils import looks_like_numpy_member, infer_numpy_member + + +def numpy_core_numeric_transform(): + return astroid.parse( + """ + # different functions defined in numeric.py + import numpy + def zeros_like(a, dtype=None, order='K', subok=True): return numpy.ndarray((0, 0)) + def ones_like(a, dtype=None, order='K', subok=True): return numpy.ndarray((0, 0)) + def full_like(a, fill_value, dtype=None, order='K', subok=True): return numpy.ndarray((0, 0)) + """ + ) + + +astroid.register_module_extender( + astroid.MANAGER, "numpy.core.numeric", numpy_core_numeric_transform +) + + +METHODS_TO_BE_INFERRED = { + "ones": """def ones(shape, dtype=None, order='C'): + return numpy.ndarray([0, 0])""" +} + + +for method_name, function_src in METHODS_TO_BE_INFERRED.items(): + inference_function = functools.partial(infer_numpy_member, function_src) + astroid.MANAGER.register_transform( + astroid.Attribute, + astroid.inference_tip(inference_function), + functools.partial(looks_like_numpy_member, method_name), + ) diff --git a/WebCrawler/venv/Lib/site-packages/astroid/brain/brain_numpy_core_numerictypes.py b/WebCrawler/venv/Lib/site-packages/astroid/brain/brain_numpy_core_numerictypes.py new file mode 100644 index 0000000..6ac4a14 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/astroid/brain/brain_numpy_core_numerictypes.py @@ -0,0 +1,254 @@ +# Copyright (c) 2019-2020 hippo91 + +# Licensed under the LGPL: https://www.gnu.org/licenses/old-licenses/lgpl-2.1.en.html +# For details: https://github.com/PyCQA/astroid/blob/master/COPYING.LESSER + +# TODO(hippo91) : correct the methods signature. + +"""Astroid hooks for numpy.core.numerictypes module.""" + +import astroid + + +def numpy_core_numerictypes_transform(): + # TODO: Uniformize the generic API with the ndarray one. + # According to numpy doc the generic object should expose + # the same API than ndarray. This has been done here partially + # through the astype method. + return astroid.parse( + """ + # different types defined in numerictypes.py + class generic(object): + def __init__(self, value): + self.T = None + self.base = None + self.data = None + self.dtype = None + self.flags = None + self.flat = None + self.imag = None + self.itemsize = None + self.nbytes = None + self.ndim = None + self.real = None + self.size = None + self.strides = None + + def all(self): return uninferable + def any(self): return uninferable + def argmax(self): return uninferable + def argmin(self): return uninferable + def argsort(self): return uninferable + def astype(self, dtype, order='K', casting='unsafe', subok=True, copy=True): return np.ndarray([0, 0]) + def base(self): return uninferable + def byteswap(self): return uninferable + def choose(self): return uninferable + def clip(self): return uninferable + def compress(self): return uninferable + def conj(self): return uninferable + def conjugate(self): return uninferable + def copy(self): return uninferable + def cumprod(self): return uninferable + def cumsum(self): return uninferable + def data(self): return uninferable + def diagonal(self): return uninferable + def dtype(self): return uninferable + def dump(self): return uninferable + def dumps(self): return uninferable + def fill(self): return uninferable + def flags(self): return uninferable + def flat(self): return uninferable + def flatten(self): return uninferable + def getfield(self): return uninferable + def imag(self): return uninferable + def item(self): return uninferable + def itemset(self): return uninferable + def itemsize(self): return uninferable + def max(self): return uninferable + def mean(self): return uninferable + def min(self): return uninferable + def nbytes(self): return uninferable + def ndim(self): return uninferable + def newbyteorder(self): return uninferable + def nonzero(self): return uninferable + def prod(self): return uninferable + def ptp(self): return uninferable + def put(self): return uninferable + def ravel(self): return uninferable + def real(self): return uninferable + def repeat(self): return uninferable + def reshape(self): return uninferable + def resize(self): return uninferable + def round(self): return uninferable + def searchsorted(self): return uninferable + def setfield(self): return uninferable + def setflags(self): return uninferable + def shape(self): return uninferable + def size(self): return uninferable + def sort(self): return uninferable + def squeeze(self): return uninferable + def std(self): return uninferable + def strides(self): return uninferable + def sum(self): return uninferable + def swapaxes(self): return uninferable + def take(self): return uninferable + def tobytes(self): return uninferable + def tofile(self): return uninferable + def tolist(self): return uninferable + def tostring(self): return uninferable + def trace(self): return uninferable + def transpose(self): return uninferable + def var(self): return uninferable + def view(self): return uninferable + + + class dtype(object): + def __init__(self, obj, align=False, copy=False): + self.alignment = None + self.base = None + self.byteorder = None + self.char = None + self.descr = None + self.fields = None + self.flags = None + self.hasobject = None + self.isalignedstruct = None + self.isbuiltin = None + self.isnative = None + self.itemsize = None + self.kind = None + self.metadata = None + self.name = None + self.names = None + self.num = None + self.shape = None + self.str = None + self.subdtype = None + self.type = None + + def newbyteorder(self, new_order='S'): return uninferable + def __neg__(self): return uninferable + + class busdaycalendar(object): + def __init__(self, weekmask='1111100', holidays=None): + self.holidays = None + self.weekmask = None + + class flexible(generic): pass + class bool_(generic): pass + class number(generic): + def __neg__(self): return uninferable + class datetime64(generic): + def __init__(self, nb, unit=None): pass + + + class void(flexible): + def __init__(self, *args, **kwargs): + self.base = None + self.dtype = None + self.flags = None + def getfield(self): return uninferable + def setfield(self): return uninferable + + + class character(flexible): pass + + + class integer(number): + def __init__(self, value): + self.denominator = None + self.numerator = None + + + class inexact(number): pass + + + class str_(str, character): + def maketrans(self, x, y=None, z=None): return uninferable + + + class bytes_(bytes, character): + def fromhex(self, string): return uninferable + def maketrans(self, frm, to): return uninferable + + + class signedinteger(integer): pass + + + class unsignedinteger(integer): pass + + + class complexfloating(inexact): pass + + + class floating(inexact): pass + + + class float64(floating, float): + def fromhex(self, string): return uninferable + + + class uint64(unsignedinteger): pass + class complex64(complexfloating): pass + class int16(signedinteger): pass + class float96(floating): pass + class int8(signedinteger): pass + class uint32(unsignedinteger): pass + class uint8(unsignedinteger): pass + class _typedict(dict): pass + class complex192(complexfloating): pass + class timedelta64(signedinteger): + def __init__(self, nb, unit=None): pass + class int32(signedinteger): pass + class uint16(unsignedinteger): pass + class float32(floating): pass + class complex128(complexfloating, complex): pass + class float16(floating): pass + class int64(signedinteger): pass + + buffer_type = memoryview + bool8 = bool_ + byte = int8 + bytes0 = bytes_ + cdouble = complex128 + cfloat = complex128 + clongdouble = complex192 + clongfloat = complex192 + complex_ = complex128 + csingle = complex64 + double = float64 + float_ = float64 + half = float16 + int0 = int32 + int_ = int32 + intc = int32 + intp = int32 + long = int32 + longcomplex = complex192 + longdouble = float96 + longfloat = float96 + longlong = int64 + object0 = object_ + object_ = object_ + short = int16 + single = float32 + singlecomplex = complex64 + str0 = str_ + string_ = bytes_ + ubyte = uint8 + uint = uint32 + uint0 = uint32 + uintc = uint32 + uintp = uint32 + ulonglong = uint64 + unicode = str_ + unicode_ = str_ + ushort = uint16 + void0 = void + """ + ) + + +astroid.register_module_extender( + astroid.MANAGER, "numpy.core.numerictypes", numpy_core_numerictypes_transform +) diff --git a/WebCrawler/venv/Lib/site-packages/astroid/brain/brain_numpy_core_umath.py b/WebCrawler/venv/Lib/site-packages/astroid/brain/brain_numpy_core_umath.py new file mode 100644 index 0000000..897961e --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/astroid/brain/brain_numpy_core_umath.py @@ -0,0 +1,147 @@ +# Copyright (c) 2019 hippo91 + +# Licensed under the LGPL: https://www.gnu.org/licenses/old-licenses/lgpl-2.1.en.html +# For details: https://github.com/PyCQA/astroid/blob/master/COPYING.LESSER + + +"""Astroid hooks for numpy.core.umath module.""" + +import astroid + + +def numpy_core_umath_transform(): + ufunc_optional_keyword_arguments = ( + """out=None, where=True, casting='same_kind', order='K', """ + """dtype=None, subok=True""" + ) + return astroid.parse( + """ + class FakeUfunc: + def __init__(self): + self.__doc__ = str() + self.__name__ = str() + self.nin = 0 + self.nout = 0 + self.nargs = 0 + self.ntypes = 0 + self.types = None + self.identity = None + self.signature = None + + @classmethod + def reduce(cls, a, axis=None, dtype=None, out=None): + return numpy.ndarray([0, 0]) + + @classmethod + def accumulate(cls, array, axis=None, dtype=None, out=None): + return numpy.ndarray([0, 0]) + + @classmethod + def reduceat(cls, a, indices, axis=None, dtype=None, out=None): + return numpy.ndarray([0, 0]) + + @classmethod + def outer(cls, A, B, **kwargs): + return numpy.ndarray([0, 0]) + + @classmethod + def at(cls, a, indices, b=None): + return numpy.ndarray([0, 0]) + + class FakeUfuncOneArg(FakeUfunc): + def __call__(self, x, {opt_args:s}): + return numpy.ndarray([0, 0]) + + class FakeUfuncOneArgBis(FakeUfunc): + def __call__(self, x, {opt_args:s}): + return numpy.ndarray([0, 0]), numpy.ndarray([0, 0]) + + class FakeUfuncTwoArgs(FakeUfunc): + def __call__(self, x1, x2, {opt_args:s}): + return numpy.ndarray([0, 0]) + + # Constants + e = 2.718281828459045 + euler_gamma = 0.5772156649015329 + + # One arg functions with optional kwargs + arccos = FakeUfuncOneArg() + arccosh = FakeUfuncOneArg() + arcsin = FakeUfuncOneArg() + arcsinh = FakeUfuncOneArg() + arctan = FakeUfuncOneArg() + arctanh = FakeUfuncOneArg() + cbrt = FakeUfuncOneArg() + conj = FakeUfuncOneArg() + conjugate = FakeUfuncOneArg() + cosh = FakeUfuncOneArg() + deg2rad = FakeUfuncOneArg() + exp2 = FakeUfuncOneArg() + expm1 = FakeUfuncOneArg() + fabs = FakeUfuncOneArg() + frexp = FakeUfuncOneArgBis() + isfinite = FakeUfuncOneArg() + isinf = FakeUfuncOneArg() + log = FakeUfuncOneArg() + log1p = FakeUfuncOneArg() + log2 = FakeUfuncOneArg() + logical_not = FakeUfuncOneArg() + modf = FakeUfuncOneArgBis() + negative = FakeUfuncOneArg() + positive = FakeUfuncOneArg() + rad2deg = FakeUfuncOneArg() + reciprocal = FakeUfuncOneArg() + rint = FakeUfuncOneArg() + sign = FakeUfuncOneArg() + signbit = FakeUfuncOneArg() + sinh = FakeUfuncOneArg() + spacing = FakeUfuncOneArg() + square = FakeUfuncOneArg() + tan = FakeUfuncOneArg() + tanh = FakeUfuncOneArg() + trunc = FakeUfuncOneArg() + + # Two args functions with optional kwargs + bitwise_and = FakeUfuncTwoArgs() + bitwise_or = FakeUfuncTwoArgs() + bitwise_xor = FakeUfuncTwoArgs() + copysign = FakeUfuncTwoArgs() + divide = FakeUfuncTwoArgs() + divmod = FakeUfuncTwoArgs() + equal = FakeUfuncTwoArgs() + float_power = FakeUfuncTwoArgs() + floor_divide = FakeUfuncTwoArgs() + fmax = FakeUfuncTwoArgs() + fmin = FakeUfuncTwoArgs() + fmod = FakeUfuncTwoArgs() + greater = FakeUfuncTwoArgs() + gcd = FakeUfuncTwoArgs() + hypot = FakeUfuncTwoArgs() + heaviside = FakeUfuncTwoArgs() + lcm = FakeUfuncTwoArgs() + ldexp = FakeUfuncTwoArgs() + left_shift = FakeUfuncTwoArgs() + less = FakeUfuncTwoArgs() + logaddexp = FakeUfuncTwoArgs() + logaddexp2 = FakeUfuncTwoArgs() + logical_and = FakeUfuncTwoArgs() + logical_or = FakeUfuncTwoArgs() + logical_xor = FakeUfuncTwoArgs() + maximum = FakeUfuncTwoArgs() + minimum = FakeUfuncTwoArgs() + nextafter = FakeUfuncTwoArgs() + not_equal = FakeUfuncTwoArgs() + power = FakeUfuncTwoArgs() + remainder = FakeUfuncTwoArgs() + right_shift = FakeUfuncTwoArgs() + subtract = FakeUfuncTwoArgs() + true_divide = FakeUfuncTwoArgs() + """.format( + opt_args=ufunc_optional_keyword_arguments + ) + ) + + +astroid.register_module_extender( + astroid.MANAGER, "numpy.core.umath", numpy_core_umath_transform +) diff --git a/WebCrawler/venv/Lib/site-packages/astroid/brain/brain_numpy_ndarray.py b/WebCrawler/venv/Lib/site-packages/astroid/brain/brain_numpy_ndarray.py new file mode 100644 index 0000000..d40a7dd --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/astroid/brain/brain_numpy_ndarray.py @@ -0,0 +1,153 @@ +# Copyright (c) 2015-2016, 2018-2019 Claudiu Popa +# Copyright (c) 2016 Ceridwen +# Copyright (c) 2017-2020 hippo91 + +# Licensed under the LGPL: https://www.gnu.org/licenses/old-licenses/lgpl-2.1.en.html +# For details: https://github.com/PyCQA/astroid/blob/master/COPYING.LESSER + + +"""Astroid hooks for numpy ndarray class.""" + +import functools +import astroid + + +def infer_numpy_ndarray(node, context=None): + ndarray = """ + class ndarray(object): + def __init__(self, shape, dtype=float, buffer=None, offset=0, + strides=None, order=None): + self.T = None + self.base = None + self.ctypes = None + self.data = None + self.dtype = None + self.flags = None + self.flat = None + self.imag = np.ndarray([0, 0]) + self.itemsize = None + self.nbytes = None + self.ndim = None + self.real = np.ndarray([0, 0]) + self.shape = numpy.ndarray([0, 0]) + self.size = None + self.strides = None + + def __abs__(self): return numpy.ndarray([0, 0]) + def __add__(self, value): return numpy.ndarray([0, 0]) + def __and__(self, value): return numpy.ndarray([0, 0]) + def __array__(self, dtype=None): return numpy.ndarray([0, 0]) + def __array_wrap__(self, obj): return numpy.ndarray([0, 0]) + def __contains__(self, key): return True + def __copy__(self): return numpy.ndarray([0, 0]) + def __deepcopy__(self, memo): return numpy.ndarray([0, 0]) + def __divmod__(self, value): return (numpy.ndarray([0, 0]), numpy.ndarray([0, 0])) + def __eq__(self, value): return numpy.ndarray([0, 0]) + def __float__(self): return 0. + def __floordiv__(self): return numpy.ndarray([0, 0]) + def __ge__(self, value): return numpy.ndarray([0, 0]) + def __getitem__(self, key): return uninferable + def __gt__(self, value): return numpy.ndarray([0, 0]) + def __iadd__(self, value): return numpy.ndarray([0, 0]) + def __iand__(self, value): return numpy.ndarray([0, 0]) + def __ifloordiv__(self, value): return numpy.ndarray([0, 0]) + def __ilshift__(self, value): return numpy.ndarray([0, 0]) + def __imod__(self, value): return numpy.ndarray([0, 0]) + def __imul__(self, value): return numpy.ndarray([0, 0]) + def __int__(self): return 0 + def __invert__(self): return numpy.ndarray([0, 0]) + def __ior__(self, value): return numpy.ndarray([0, 0]) + def __ipow__(self, value): return numpy.ndarray([0, 0]) + def __irshift__(self, value): return numpy.ndarray([0, 0]) + def __isub__(self, value): return numpy.ndarray([0, 0]) + def __itruediv__(self, value): return numpy.ndarray([0, 0]) + def __ixor__(self, value): return numpy.ndarray([0, 0]) + def __le__(self, value): return numpy.ndarray([0, 0]) + def __len__(self): return 1 + def __lshift__(self, value): return numpy.ndarray([0, 0]) + def __lt__(self, value): return numpy.ndarray([0, 0]) + def __matmul__(self, value): return numpy.ndarray([0, 0]) + def __mod__(self, value): return numpy.ndarray([0, 0]) + def __mul__(self, value): return numpy.ndarray([0, 0]) + def __ne__(self, value): return numpy.ndarray([0, 0]) + def __neg__(self): return numpy.ndarray([0, 0]) + def __or__(self): return numpy.ndarray([0, 0]) + def __pos__(self): return numpy.ndarray([0, 0]) + def __pow__(self): return numpy.ndarray([0, 0]) + def __repr__(self): return str() + def __rshift__(self): return numpy.ndarray([0, 0]) + def __setitem__(self, key, value): return uninferable + def __str__(self): return str() + def __sub__(self, value): return numpy.ndarray([0, 0]) + def __truediv__(self, value): return numpy.ndarray([0, 0]) + def __xor__(self, value): return numpy.ndarray([0, 0]) + def all(self, axis=None, out=None, keepdims=False): return np.ndarray([0, 0]) + def any(self, axis=None, out=None, keepdims=False): return np.ndarray([0, 0]) + def argmax(self, axis=None, out=None): return np.ndarray([0, 0]) + def argmin(self, axis=None, out=None): return np.ndarray([0, 0]) + def argpartition(self, kth, axis=-1, kind='introselect', order=None): return np.ndarray([0, 0]) + def argsort(self, axis=-1, kind='quicksort', order=None): return np.ndarray([0, 0]) + def astype(self, dtype, order='K', casting='unsafe', subok=True, copy=True): return np.ndarray([0, 0]) + def byteswap(self, inplace=False): return np.ndarray([0, 0]) + def choose(self, choices, out=None, mode='raise'): return np.ndarray([0, 0]) + def clip(self, min=None, max=None, out=None): return np.ndarray([0, 0]) + def compress(self, condition, axis=None, out=None): return np.ndarray([0, 0]) + def conj(self): return np.ndarray([0, 0]) + def conjugate(self): return np.ndarray([0, 0]) + def copy(self, order='C'): return np.ndarray([0, 0]) + def cumprod(self, axis=None, dtype=None, out=None): return np.ndarray([0, 0]) + def cumsum(self, axis=None, dtype=None, out=None): return np.ndarray([0, 0]) + def diagonal(self, offset=0, axis1=0, axis2=1): return np.ndarray([0, 0]) + def dot(self, b, out=None): return np.ndarray([0, 0]) + def dump(self, file): return None + def dumps(self): return str() + def fill(self, value): return None + def flatten(self, order='C'): return np.ndarray([0, 0]) + def getfield(self, dtype, offset=0): return np.ndarray([0, 0]) + def item(self, *args): return uninferable + def itemset(self, *args): return None + def max(self, axis=None, out=None): return np.ndarray([0, 0]) + def mean(self, axis=None, dtype=None, out=None, keepdims=False): return np.ndarray([0, 0]) + def min(self, axis=None, out=None, keepdims=False): return np.ndarray([0, 0]) + def newbyteorder(self, new_order='S'): return np.ndarray([0, 0]) + def nonzero(self): return (1,) + def partition(self, kth, axis=-1, kind='introselect', order=None): return None + def prod(self, axis=None, dtype=None, out=None, keepdims=False): return np.ndarray([0, 0]) + def ptp(self, axis=None, out=None): return np.ndarray([0, 0]) + def put(self, indices, values, mode='raise'): return None + def ravel(self, order='C'): return np.ndarray([0, 0]) + def repeat(self, repeats, axis=None): return np.ndarray([0, 0]) + def reshape(self, shape, order='C'): return np.ndarray([0, 0]) + def resize(self, new_shape, refcheck=True): return None + def round(self, decimals=0, out=None): return np.ndarray([0, 0]) + def searchsorted(self, v, side='left', sorter=None): return np.ndarray([0, 0]) + def setfield(self, val, dtype, offset=0): return None + def setflags(self, write=None, align=None, uic=None): return None + def sort(self, axis=-1, kind='quicksort', order=None): return None + def squeeze(self, axis=None): return np.ndarray([0, 0]) + def std(self, axis=None, dtype=None, out=None, ddof=0, keepdims=False): return np.ndarray([0, 0]) + def sum(self, axis=None, dtype=None, out=None, keepdims=False): return np.ndarray([0, 0]) + def swapaxes(self, axis1, axis2): return np.ndarray([0, 0]) + def take(self, indices, axis=None, out=None, mode='raise'): return np.ndarray([0, 0]) + def tobytes(self, order='C'): return b'' + def tofile(self, fid, sep="", format="%s"): return None + def tolist(self, ): return [] + def tostring(self, order='C'): return b'' + def trace(self, offset=0, axis1=0, axis2=1, dtype=None, out=None): return np.ndarray([0, 0]) + def transpose(self, *axes): return np.ndarray([0, 0]) + def var(self, axis=None, dtype=None, out=None, ddof=0, keepdims=False): return np.ndarray([0, 0]) + def view(self, dtype=None, type=None): return np.ndarray([0, 0]) + """ + node = astroid.extract_node(ndarray) + return node.infer(context=context) + + +def _looks_like_numpy_ndarray(node): + return isinstance(node, astroid.Attribute) and node.attrname == "ndarray" + + +astroid.MANAGER.register_transform( + astroid.Attribute, + astroid.inference_tip(infer_numpy_ndarray), + _looks_like_numpy_ndarray, +) diff --git a/WebCrawler/venv/Lib/site-packages/astroid/brain/brain_numpy_random_mtrand.py b/WebCrawler/venv/Lib/site-packages/astroid/brain/brain_numpy_random_mtrand.py new file mode 100644 index 0000000..cffdcee --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/astroid/brain/brain_numpy_random_mtrand.py @@ -0,0 +1,70 @@ +# Copyright (c) 2019 hippo91 + +# Licensed under the LGPL: https://www.gnu.org/licenses/old-licenses/lgpl-2.1.en.html +# For details: https://github.com/PyCQA/astroid/blob/master/COPYING.LESSER + +# TODO(hippo91) : correct the functions return types +"""Astroid hooks for numpy.random.mtrand module.""" + +import astroid + + +def numpy_random_mtrand_transform(): + return astroid.parse( + """ + def beta(a, b, size=None): return uninferable + def binomial(n, p, size=None): return uninferable + def bytes(length): return uninferable + def chisquare(df, size=None): return uninferable + def choice(a, size=None, replace=True, p=None): return uninferable + def dirichlet(alpha, size=None): return uninferable + def exponential(scale=1.0, size=None): return uninferable + def f(dfnum, dfden, size=None): return uninferable + def gamma(shape, scale=1.0, size=None): return uninferable + def geometric(p, size=None): return uninferable + def get_state(): return uninferable + def gumbel(loc=0.0, scale=1.0, size=None): return uninferable + def hypergeometric(ngood, nbad, nsample, size=None): return uninferable + def laplace(loc=0.0, scale=1.0, size=None): return uninferable + def logistic(loc=0.0, scale=1.0, size=None): return uninferable + def lognormal(mean=0.0, sigma=1.0, size=None): return uninferable + def logseries(p, size=None): return uninferable + def multinomial(n, pvals, size=None): return uninferable + def multivariate_normal(mean, cov, size=None): return uninferable + def negative_binomial(n, p, size=None): return uninferable + def noncentral_chisquare(df, nonc, size=None): return uninferable + def noncentral_f(dfnum, dfden, nonc, size=None): return uninferable + def normal(loc=0.0, scale=1.0, size=None): return uninferable + def pareto(a, size=None): return uninferable + def permutation(x): return uninferable + def poisson(lam=1.0, size=None): return uninferable + def power(a, size=None): return uninferable + def rand(*args): return uninferable + def randint(low, high=None, size=None, dtype='l'): + import numpy + return numpy.ndarray((1,1)) + def randn(*args): return uninferable + def random_integers(low, high=None, size=None): return uninferable + def random_sample(size=None): return uninferable + def rayleigh(scale=1.0, size=None): return uninferable + def seed(seed=None): return uninferable + def set_state(state): return uninferable + def shuffle(x): return uninferable + def standard_cauchy(size=None): return uninferable + def standard_exponential(size=None): return uninferable + def standard_gamma(shape, size=None): return uninferable + def standard_normal(size=None): return uninferable + def standard_t(df, size=None): return uninferable + def triangular(left, mode, right, size=None): return uninferable + def uniform(low=0.0, high=1.0, size=None): return uninferable + def vonmises(mu, kappa, size=None): return uninferable + def wald(mean, scale, size=None): return uninferable + def weibull(a, size=None): return uninferable + def zipf(a, size=None): return uninferable + """ + ) + + +astroid.register_module_extender( + astroid.MANAGER, "numpy.random.mtrand", numpy_random_mtrand_transform +) diff --git a/WebCrawler/venv/Lib/site-packages/astroid/brain/brain_numpy_utils.py b/WebCrawler/venv/Lib/site-packages/astroid/brain/brain_numpy_utils.py new file mode 100644 index 0000000..b29d271 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/astroid/brain/brain_numpy_utils.py @@ -0,0 +1,65 @@ +# Copyright (c) 2019-2020 hippo91 +# Copyright (c) 2019 Claudiu Popa + +# Licensed under the LGPL: https://www.gnu.org/licenses/old-licenses/lgpl-2.1.en.html +# For details: https://github.com/PyCQA/astroid/blob/master/COPYING.LESSER + + +"""Different utilities for the numpy brains""" + + +import astroid + + +def infer_numpy_member(src, node, context=None): + node = astroid.extract_node(src) + return node.infer(context=context) + + +def _is_a_numpy_module(node: astroid.node_classes.Name) -> bool: + """ + Returns True if the node is a representation of a numpy module. + + For example in : + import numpy as np + x = np.linspace(1, 2) + The node is a representation of the numpy module. + + :param node: node to test + :return: True if the node is a representation of the numpy module. + """ + module_nickname = node.name + potential_import_target = [ + x for x in node.lookup(module_nickname)[1] if isinstance(x, astroid.Import) + ] + for target in potential_import_target: + if ("numpy", module_nickname) in target.names: + return True + return False + + +def looks_like_numpy_member( + member_name: str, node: astroid.node_classes.NodeNG +) -> bool: + """ + Returns True if the node is a member of numpy whose + name is member_name. + + :param member_name: name of the member + :param node: node to test + :return: True if the node is a member of numpy + """ + if ( + isinstance(node, astroid.Attribute) + and node.attrname == member_name + and isinstance(node.expr, astroid.Name) + and _is_a_numpy_module(node.expr) + ): + return True + if ( + isinstance(node, astroid.Name) + and node.name == member_name + and node.root().name.startswith("numpy") + ): + return True + return False diff --git a/WebCrawler/venv/Lib/site-packages/astroid/brain/brain_pkg_resources.py b/WebCrawler/venv/Lib/site-packages/astroid/brain/brain_pkg_resources.py new file mode 100644 index 0000000..25e7649 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/astroid/brain/brain_pkg_resources.py @@ -0,0 +1,75 @@ +# Copyright (c) 2016, 2018 Claudiu Popa +# Copyright (c) 2016 Ceridwen + +# Licensed under the LGPL: https://www.gnu.org/licenses/old-licenses/lgpl-2.1.en.html +# For details: https://github.com/PyCQA/astroid/blob/master/COPYING.LESSER + + +import astroid +from astroid import parse +from astroid import inference_tip +from astroid import register_module_extender +from astroid import MANAGER + + +def pkg_resources_transform(): + return parse( + """ +def require(*requirements): + return pkg_resources.working_set.require(*requirements) + +def run_script(requires, script_name): + return pkg_resources.working_set.run_script(requires, script_name) + +def iter_entry_points(group, name=None): + return pkg_resources.working_set.iter_entry_points(group, name) + +def resource_exists(package_or_requirement, resource_name): + return get_provider(package_or_requirement).has_resource(resource_name) + +def resource_isdir(package_or_requirement, resource_name): + return get_provider(package_or_requirement).resource_isdir( + resource_name) + +def resource_filename(package_or_requirement, resource_name): + return get_provider(package_or_requirement).get_resource_filename( + self, resource_name) + +def resource_stream(package_or_requirement, resource_name): + return get_provider(package_or_requirement).get_resource_stream( + self, resource_name) + +def resource_string(package_or_requirement, resource_name): + return get_provider(package_or_requirement).get_resource_string( + self, resource_name) + +def resource_listdir(package_or_requirement, resource_name): + return get_provider(package_or_requirement).resource_listdir( + resource_name) + +def extraction_error(): + pass + +def get_cache_path(archive_name, names=()): + extract_path = self.extraction_path or get_default_cache() + target_path = os.path.join(extract_path, archive_name+'-tmp', *names) + return target_path + +def postprocess(tempname, filename): + pass + +def set_extraction_path(path): + pass + +def cleanup_resources(force=False): + pass + +def get_distribution(dist): + return Distribution(dist) + +_namespace_packages = {} +""" + ) + + +register_module_extender(MANAGER, "pkg_resources", pkg_resources_transform) diff --git a/WebCrawler/venv/Lib/site-packages/astroid/brain/brain_pytest.py b/WebCrawler/venv/Lib/site-packages/astroid/brain/brain_pytest.py new file mode 100644 index 0000000..56202ab --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/astroid/brain/brain_pytest.py @@ -0,0 +1,88 @@ +# Copyright (c) 2014-2016, 2018 Claudiu Popa +# Copyright (c) 2014 Jeff Quast +# Copyright (c) 2014 Google, Inc. +# Copyright (c) 2016 Florian Bruhin +# Copyright (c) 2016 Ceridwen + +# Licensed under the LGPL: https://www.gnu.org/licenses/old-licenses/lgpl-2.1.en.html +# For details: https://github.com/PyCQA/astroid/blob/master/COPYING.LESSER + +"""Astroid hooks for pytest.""" +from __future__ import absolute_import +from astroid import MANAGER, register_module_extender +from astroid.builder import AstroidBuilder + + +def pytest_transform(): + return AstroidBuilder(MANAGER).string_build( + """ + +try: + import _pytest.mark + import _pytest.recwarn + import _pytest.runner + import _pytest.python + import _pytest.skipping + import _pytest.assertion +except ImportError: + pass +else: + deprecated_call = _pytest.recwarn.deprecated_call + warns = _pytest.recwarn.warns + + exit = _pytest.runner.exit + fail = _pytest.runner.fail + skip = _pytest.runner.skip + importorskip = _pytest.runner.importorskip + + xfail = _pytest.skipping.xfail + mark = _pytest.mark.MarkGenerator() + raises = _pytest.python.raises + + # New in pytest 3.0 + try: + approx = _pytest.python.approx + register_assert_rewrite = _pytest.assertion.register_assert_rewrite + except AttributeError: + pass + + +# Moved in pytest 3.0 + +try: + import _pytest.freeze_support + freeze_includes = _pytest.freeze_support.freeze_includes +except ImportError: + try: + import _pytest.genscript + freeze_includes = _pytest.genscript.freeze_includes + except ImportError: + pass + +try: + import _pytest.debugging + set_trace = _pytest.debugging.pytestPDB().set_trace +except ImportError: + try: + import _pytest.pdb + set_trace = _pytest.pdb.pytestPDB().set_trace + except ImportError: + pass + +try: + import _pytest.fixtures + fixture = _pytest.fixtures.fixture + yield_fixture = _pytest.fixtures.yield_fixture +except ImportError: + try: + import _pytest.python + fixture = _pytest.python.fixture + yield_fixture = _pytest.python.yield_fixture + except ImportError: + pass +""" + ) + + +register_module_extender(MANAGER, "pytest", pytest_transform) +register_module_extender(MANAGER, "py.test", pytest_transform) diff --git a/WebCrawler/venv/Lib/site-packages/astroid/brain/brain_qt.py b/WebCrawler/venv/Lib/site-packages/astroid/brain/brain_qt.py new file mode 100644 index 0000000..b703b37 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/astroid/brain/brain_qt.py @@ -0,0 +1,83 @@ +# Copyright (c) 2015-2016, 2018 Claudiu Popa +# Copyright (c) 2016 Ceridwen +# Copyright (c) 2017 Roy Wright +# Copyright (c) 2018 Ashley Whetter +# Copyright (c) 2019 Antoine Boellinger + +# Licensed under the LGPL: https://www.gnu.org/licenses/old-licenses/lgpl-2.1.en.html +# For details: https://github.com/PyCQA/astroid/blob/master/COPYING.LESSER + +"""Astroid hooks for the PyQT library.""" + +from astroid import MANAGER, register_module_extender +from astroid.builder import AstroidBuilder +from astroid import nodes +from astroid import parse + + +def _looks_like_signal(node, signal_name="pyqtSignal"): + if "__class__" in node.instance_attrs: + try: + cls = node.instance_attrs["__class__"][0] + return cls.name == signal_name + except AttributeError: + # return False if the cls does not have a name attribute + pass + return False + + +def transform_pyqt_signal(node): + module = parse( + """ + class pyqtSignal(object): + def connect(self, slot, type=None, no_receiver_check=False): + pass + def disconnect(self, slot): + pass + def emit(self, *args): + pass + """ + ) + signal_cls = module["pyqtSignal"] + node.instance_attrs["emit"] = signal_cls["emit"] + node.instance_attrs["disconnect"] = signal_cls["disconnect"] + node.instance_attrs["connect"] = signal_cls["connect"] + + +def transform_pyside_signal(node): + module = parse( + """ + class NotPySideSignal(object): + def connect(self, receiver, type=None): + pass + def disconnect(self, receiver): + pass + def emit(self, *args): + pass + """ + ) + signal_cls = module["NotPySideSignal"] + node.instance_attrs["connect"] = signal_cls["connect"] + node.instance_attrs["disconnect"] = signal_cls["disconnect"] + node.instance_attrs["emit"] = signal_cls["emit"] + + +def pyqt4_qtcore_transform(): + return AstroidBuilder(MANAGER).string_build( + """ + +def SIGNAL(signal_name): pass + +class QObject(object): + def emit(self, signal): pass +""" + ) + + +register_module_extender(MANAGER, "PyQt4.QtCore", pyqt4_qtcore_transform) +MANAGER.register_transform(nodes.FunctionDef, transform_pyqt_signal, _looks_like_signal) +MANAGER.register_transform( + nodes.ClassDef, + transform_pyside_signal, + lambda node: node.qname() in ("PySide.QtCore.Signal", "PySide2.QtCore.Signal"), +) diff --git a/WebCrawler/venv/Lib/site-packages/astroid/brain/brain_random.py b/WebCrawler/venv/Lib/site-packages/astroid/brain/brain_random.py new file mode 100644 index 0000000..5ec858a --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/astroid/brain/brain_random.py @@ -0,0 +1,75 @@ +# Licensed under the LGPL: https://www.gnu.org/licenses/old-licenses/lgpl-2.1.en.html +# For details: https://github.com/PyCQA/astroid/blob/master/COPYING.LESSER +import random + +import astroid +from astroid import helpers +from astroid import MANAGER + + +ACCEPTED_ITERABLES_FOR_SAMPLE = (astroid.List, astroid.Set, astroid.Tuple) + + +def _clone_node_with_lineno(node, parent, lineno): + cls = node.__class__ + other_fields = node._other_fields + _astroid_fields = node._astroid_fields + init_params = {"lineno": lineno, "col_offset": node.col_offset, "parent": parent} + postinit_params = {param: getattr(node, param) for param in _astroid_fields} + if other_fields: + init_params.update({param: getattr(node, param) for param in other_fields}) + new_node = cls(**init_params) + if hasattr(node, "postinit") and _astroid_fields: + new_node.postinit(**postinit_params) + return new_node + + +def infer_random_sample(node, context=None): + if len(node.args) != 2: + raise astroid.UseInferenceDefault + + length = node.args[1] + if not isinstance(length, astroid.Const): + raise astroid.UseInferenceDefault + if not isinstance(length.value, int): + raise astroid.UseInferenceDefault + + inferred_sequence = helpers.safe_infer(node.args[0], context=context) + if not inferred_sequence: + raise astroid.UseInferenceDefault + + if not isinstance(inferred_sequence, ACCEPTED_ITERABLES_FOR_SAMPLE): + raise astroid.UseInferenceDefault + + if length.value > len(inferred_sequence.elts): + # In this case, this will raise a ValueError + raise astroid.UseInferenceDefault + + try: + elts = random.sample(inferred_sequence.elts, length.value) + except ValueError: + raise astroid.UseInferenceDefault + + new_node = astroid.List( + lineno=node.lineno, col_offset=node.col_offset, parent=node.scope() + ) + new_elts = [ + _clone_node_with_lineno(elt, parent=new_node, lineno=new_node.lineno) + for elt in elts + ] + new_node.postinit(new_elts) + return iter((new_node,)) + + +def _looks_like_random_sample(node): + func = node.func + if isinstance(func, astroid.Attribute): + return func.attrname == "sample" + if isinstance(func, astroid.Name): + return func.name == "sample" + return False + + +MANAGER.register_transform( + astroid.Call, astroid.inference_tip(infer_random_sample), _looks_like_random_sample +) diff --git a/WebCrawler/venv/Lib/site-packages/astroid/brain/brain_re.py b/WebCrawler/venv/Lib/site-packages/astroid/brain/brain_re.py new file mode 100644 index 0000000..c7ee51a --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/astroid/brain/brain_re.py @@ -0,0 +1,36 @@ +# Licensed under the LGPL: https://www.gnu.org/licenses/old-licenses/lgpl-2.1.en.html +# For details: https://github.com/PyCQA/astroid/blob/master/COPYING.LESSER +import sys +import astroid + +PY36 = sys.version_info >= (3, 6) + +if PY36: + # Since Python 3.6 there is the RegexFlag enum + # where every entry will be exposed via updating globals() + + def _re_transform(): + return astroid.parse( + """ + import sre_compile + ASCII = sre_compile.SRE_FLAG_ASCII + IGNORECASE = sre_compile.SRE_FLAG_IGNORECASE + LOCALE = sre_compile.SRE_FLAG_LOCALE + UNICODE = sre_compile.SRE_FLAG_UNICODE + MULTILINE = sre_compile.SRE_FLAG_MULTILINE + DOTALL = sre_compile.SRE_FLAG_DOTALL + VERBOSE = sre_compile.SRE_FLAG_VERBOSE + A = ASCII + I = IGNORECASE + L = LOCALE + U = UNICODE + M = MULTILINE + S = DOTALL + X = VERBOSE + TEMPLATE = sre_compile.SRE_FLAG_TEMPLATE + T = TEMPLATE + DEBUG = sre_compile.SRE_FLAG_DEBUG + """ + ) + + astroid.register_module_extender(astroid.MANAGER, "re", _re_transform) diff --git a/WebCrawler/venv/Lib/site-packages/astroid/brain/brain_responses.py b/WebCrawler/venv/Lib/site-packages/astroid/brain/brain_responses.py new file mode 100644 index 0000000..3a44129 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/astroid/brain/brain_responses.py @@ -0,0 +1,73 @@ +""" +Astroid hooks for responses. + +It might need to be manually updated from the public methods of +:class:`responses.RequestsMock`. + +See: https://github.com/getsentry/responses/blob/master/responses.py + +""" +import astroid + + +def responses_funcs(): + return astroid.parse( + """ + DELETE = "DELETE" + GET = "GET" + HEAD = "HEAD" + OPTIONS = "OPTIONS" + PATCH = "PATCH" + POST = "POST" + PUT = "PUT" + response_callback = None + + def reset(): + return + + def add( + method=None, # method or ``Response`` + url=None, + body="", + adding_headers=None, + *args, + **kwargs + ): + return + + def add_passthru(prefix): + return + + def remove(method_or_response=None, url=None): + return + + def replace(method_or_response=None, url=None, body="", *args, **kwargs): + return + + def add_callback( + method, url, callback, match_querystring=False, content_type="text/plain" + ): + return + + calls = [] + + def __enter__(): + return + + def __exit__(type, value, traceback): + success = type is None + return success + + def activate(func): + return func + + def start(): + return + + def stop(allow_assert=True): + return + """ + ) + + +astroid.register_module_extender(astroid.MANAGER, "responses", responses_funcs) diff --git a/WebCrawler/venv/Lib/site-packages/astroid/brain/brain_scipy_signal.py b/WebCrawler/venv/Lib/site-packages/astroid/brain/brain_scipy_signal.py new file mode 100644 index 0000000..4d00c6c --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/astroid/brain/brain_scipy_signal.py @@ -0,0 +1,89 @@ +# Copyright (c) 2019 Valentin Valls + +# Licensed under the LGPL: https://www.gnu.org/licenses/old-licenses/lgpl-2.1.en.html +# For details: https://github.com/PyCQA/astroid/blob/master/COPYING.LESSER + + +"""Astroid hooks for scipy.signal module.""" + +import astroid + + +def scipy_signal(): + return astroid.parse( + """ + # different functions defined in scipy.signals + + def barthann(M, sym=True): + return numpy.ndarray([0]) + + def bartlett(M, sym=True): + return numpy.ndarray([0]) + + def blackman(M, sym=True): + return numpy.ndarray([0]) + + def blackmanharris(M, sym=True): + return numpy.ndarray([0]) + + def bohman(M, sym=True): + return numpy.ndarray([0]) + + def boxcar(M, sym=True): + return numpy.ndarray([0]) + + def chebwin(M, at, sym=True): + return numpy.ndarray([0]) + + def cosine(M, sym=True): + return numpy.ndarray([0]) + + def exponential(M, center=None, tau=1.0, sym=True): + return numpy.ndarray([0]) + + def flattop(M, sym=True): + return numpy.ndarray([0]) + + def gaussian(M, std, sym=True): + return numpy.ndarray([0]) + + def general_gaussian(M, p, sig, sym=True): + return numpy.ndarray([0]) + + def hamming(M, sym=True): + return numpy.ndarray([0]) + + def hann(M, sym=True): + return numpy.ndarray([0]) + + def hanning(M, sym=True): + return numpy.ndarray([0]) + + def impulse2(system, X0=None, T=None, N=None, **kwargs): + return numpy.ndarray([0]), numpy.ndarray([0]) + + def kaiser(M, beta, sym=True): + return numpy.ndarray([0]) + + def nuttall(M, sym=True): + return numpy.ndarray([0]) + + def parzen(M, sym=True): + return numpy.ndarray([0]) + + def slepian(M, width, sym=True): + return numpy.ndarray([0]) + + def step2(system, X0=None, T=None, N=None, **kwargs): + return numpy.ndarray([0]), numpy.ndarray([0]) + + def triang(M, sym=True): + return numpy.ndarray([0]) + + def tukey(M, alpha=0.5, sym=True): + return numpy.ndarray([0]) + """ + ) + + +astroid.register_module_extender(astroid.MANAGER, "scipy.signal", scipy_signal) diff --git a/WebCrawler/venv/Lib/site-packages/astroid/brain/brain_six.py b/WebCrawler/venv/Lib/site-packages/astroid/brain/brain_six.py new file mode 100644 index 0000000..46d9fa3 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/astroid/brain/brain_six.py @@ -0,0 +1,201 @@ +# Copyright (c) 2014-2016, 2018, 2020 Claudiu Popa +# Copyright (c) 2015-2016 Ceridwen +# Copyright (c) 2018 Bryce Guinta + +# Licensed under the LGPL: https://www.gnu.org/licenses/old-licenses/lgpl-2.1.en.html +# For details: https://github.com/PyCQA/astroid/blob/master/COPYING.LESSER + + +"""Astroid hooks for six module.""" + +from textwrap import dedent + +from astroid import MANAGER, register_module_extender +from astroid.builder import AstroidBuilder +from astroid.exceptions import ( + AstroidBuildingError, + InferenceError, + AttributeInferenceError, +) +from astroid import nodes + + +SIX_ADD_METACLASS = "six.add_metaclass" + + +def _indent(text, prefix, predicate=None): + """Adds 'prefix' to the beginning of selected lines in 'text'. + + If 'predicate' is provided, 'prefix' will only be added to the lines + where 'predicate(line)' is True. If 'predicate' is not provided, + it will default to adding 'prefix' to all non-empty lines that do not + consist solely of whitespace characters. + """ + if predicate is None: + predicate = lambda line: line.strip() + + def prefixed_lines(): + for line in text.splitlines(True): + yield prefix + line if predicate(line) else line + + return "".join(prefixed_lines()) + + +_IMPORTS = """ +import _io +cStringIO = _io.StringIO +filter = filter +from itertools import filterfalse +input = input +from sys import intern +map = map +range = range +from importlib import reload +reload_module = lambda module: reload(module) +from functools import reduce +from shlex import quote as shlex_quote +from io import StringIO +from collections import UserDict, UserList, UserString +xrange = range +zip = zip +from itertools import zip_longest +import builtins +import configparser +import copyreg +import _dummy_thread +import http.cookiejar as http_cookiejar +import http.cookies as http_cookies +import html.entities as html_entities +import html.parser as html_parser +import http.client as http_client +import http.server as http_server +BaseHTTPServer = CGIHTTPServer = SimpleHTTPServer = http.server +import pickle as cPickle +import queue +import reprlib +import socketserver +import _thread +import winreg +import xmlrpc.server as xmlrpc_server +import xmlrpc.client as xmlrpc_client +import urllib.robotparser as urllib_robotparser +import email.mime.multipart as email_mime_multipart +import email.mime.nonmultipart as email_mime_nonmultipart +import email.mime.text as email_mime_text +import email.mime.base as email_mime_base +import urllib.parse as urllib_parse +import urllib.error as urllib_error +import tkinter +import tkinter.dialog as tkinter_dialog +import tkinter.filedialog as tkinter_filedialog +import tkinter.scrolledtext as tkinter_scrolledtext +import tkinter.simpledialog as tkinder_simpledialog +import tkinter.tix as tkinter_tix +import tkinter.ttk as tkinter_ttk +import tkinter.constants as tkinter_constants +import tkinter.dnd as tkinter_dnd +import tkinter.colorchooser as tkinter_colorchooser +import tkinter.commondialog as tkinter_commondialog +import tkinter.filedialog as tkinter_tkfiledialog +import tkinter.font as tkinter_font +import tkinter.messagebox as tkinter_messagebox +import urllib +import urllib.request as urllib_request +import urllib.robotparser as urllib_robotparser +import urllib.parse as urllib_parse +import urllib.error as urllib_error +""" + + +def six_moves_transform(): + code = dedent( + """ + class Moves(object): + {} + moves = Moves() + """ + ).format(_indent(_IMPORTS, " ")) + module = AstroidBuilder(MANAGER).string_build(code) + module.name = "six.moves" + return module + + +def _six_fail_hook(modname): + """Fix six.moves imports due to the dynamic nature of this + class. + + Construct a pseudo-module which contains all the necessary imports + for six + + :param modname: Name of failed module + :type modname: str + + :return: An astroid module + :rtype: nodes.Module + """ + + attribute_of = modname != "six.moves" and modname.startswith("six.moves") + if modname != "six.moves" and not attribute_of: + raise AstroidBuildingError(modname=modname) + module = AstroidBuilder(MANAGER).string_build(_IMPORTS) + module.name = "six.moves" + if attribute_of: + # Facilitate import of submodules in Moves + start_index = len(module.name) + attribute = modname[start_index:].lstrip(".").replace(".", "_") + try: + import_attr = module.getattr(attribute)[0] + except AttributeInferenceError: + raise AstroidBuildingError(modname=modname) + if isinstance(import_attr, nodes.Import): + submodule = MANAGER.ast_from_module_name(import_attr.names[0][0]) + return submodule + # Let dummy submodule imports pass through + # This will cause an Uninferable result, which is okay + return module + + +def _looks_like_decorated_with_six_add_metaclass(node): + if not node.decorators: + return False + + for decorator in node.decorators.nodes: + if not isinstance(decorator, nodes.Call): + continue + if decorator.func.as_string() == SIX_ADD_METACLASS: + return True + return False + + +def transform_six_add_metaclass(node): + """Check if the given class node is decorated with *six.add_metaclass* + + If so, inject its argument as the metaclass of the underlying class. + """ + if not node.decorators: + return + + for decorator in node.decorators.nodes: + if not isinstance(decorator, nodes.Call): + continue + + try: + func = next(decorator.func.infer()) + except InferenceError: + continue + if func.qname() == SIX_ADD_METACLASS and decorator.args: + metaclass = decorator.args[0] + node._metaclass = metaclass + return node + + +register_module_extender(MANAGER, "six", six_moves_transform) +register_module_extender( + MANAGER, "requests.packages.urllib3.packages.six", six_moves_transform +) +MANAGER.register_failed_import_hook(_six_fail_hook) +MANAGER.register_transform( + nodes.ClassDef, + transform_six_add_metaclass, + _looks_like_decorated_with_six_add_metaclass, +) diff --git a/WebCrawler/venv/Lib/site-packages/astroid/brain/brain_ssl.py b/WebCrawler/venv/Lib/site-packages/astroid/brain/brain_ssl.py new file mode 100644 index 0000000..2ae21c3 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/astroid/brain/brain_ssl.py @@ -0,0 +1,75 @@ +# Copyright (c) 2016, 2018 Claudiu Popa +# Copyright (c) 2016 Ceridwen +# Copyright (c) 2019 Benjamin Elven <25181435+S3ntinelX@users.noreply.github.com> + +# Licensed under the LGPL: https://www.gnu.org/licenses/old-licenses/lgpl-2.1.en.html +# For details: https://github.com/PyCQA/astroid/blob/master/COPYING.LESSER + +"""Astroid hooks for the ssl library.""" + +from astroid import MANAGER, register_module_extender +from astroid.builder import AstroidBuilder +from astroid import nodes +from astroid import parse + + +def ssl_transform(): + return parse( + """ + from _ssl import OPENSSL_VERSION_NUMBER, OPENSSL_VERSION_INFO, OPENSSL_VERSION + from _ssl import _SSLContext, MemoryBIO + from _ssl import ( + SSLError, SSLZeroReturnError, SSLWantReadError, SSLWantWriteError, + SSLSyscallError, SSLEOFError, + ) + from _ssl import CERT_NONE, CERT_OPTIONAL, CERT_REQUIRED + from _ssl import txt2obj as _txt2obj, nid2obj as _nid2obj + from _ssl import RAND_status, RAND_add, RAND_bytes, RAND_pseudo_bytes + try: + from _ssl import RAND_egd + except ImportError: + # LibreSSL does not provide RAND_egd + pass + from _ssl import (OP_ALL, OP_CIPHER_SERVER_PREFERENCE, + OP_NO_COMPRESSION, OP_NO_SSLv2, OP_NO_SSLv3, + OP_NO_TLSv1, OP_NO_TLSv1_1, OP_NO_TLSv1_2, + OP_SINGLE_DH_USE, OP_SINGLE_ECDH_USE) + + from _ssl import (ALERT_DESCRIPTION_ACCESS_DENIED, ALERT_DESCRIPTION_BAD_CERTIFICATE, + ALERT_DESCRIPTION_BAD_CERTIFICATE_HASH_VALUE, + ALERT_DESCRIPTION_BAD_CERTIFICATE_STATUS_RESPONSE, + ALERT_DESCRIPTION_BAD_RECORD_MAC, + ALERT_DESCRIPTION_CERTIFICATE_EXPIRED, + ALERT_DESCRIPTION_CERTIFICATE_REVOKED, + ALERT_DESCRIPTION_CERTIFICATE_UNKNOWN, + ALERT_DESCRIPTION_CERTIFICATE_UNOBTAINABLE, + ALERT_DESCRIPTION_CLOSE_NOTIFY, ALERT_DESCRIPTION_DECODE_ERROR, + ALERT_DESCRIPTION_DECOMPRESSION_FAILURE, + ALERT_DESCRIPTION_DECRYPT_ERROR, + ALERT_DESCRIPTION_HANDSHAKE_FAILURE, + ALERT_DESCRIPTION_ILLEGAL_PARAMETER, + ALERT_DESCRIPTION_INSUFFICIENT_SECURITY, + ALERT_DESCRIPTION_INTERNAL_ERROR, + ALERT_DESCRIPTION_NO_RENEGOTIATION, + ALERT_DESCRIPTION_PROTOCOL_VERSION, + ALERT_DESCRIPTION_RECORD_OVERFLOW, + ALERT_DESCRIPTION_UNEXPECTED_MESSAGE, + ALERT_DESCRIPTION_UNKNOWN_CA, + ALERT_DESCRIPTION_UNKNOWN_PSK_IDENTITY, + ALERT_DESCRIPTION_UNRECOGNIZED_NAME, + ALERT_DESCRIPTION_UNSUPPORTED_CERTIFICATE, + ALERT_DESCRIPTION_UNSUPPORTED_EXTENSION, + ALERT_DESCRIPTION_USER_CANCELLED) + from _ssl import (SSL_ERROR_EOF, SSL_ERROR_INVALID_ERROR_CODE, SSL_ERROR_SSL, + SSL_ERROR_SYSCALL, SSL_ERROR_WANT_CONNECT, SSL_ERROR_WANT_READ, + SSL_ERROR_WANT_WRITE, SSL_ERROR_WANT_X509_LOOKUP, SSL_ERROR_ZERO_RETURN) + from _ssl import VERIFY_CRL_CHECK_CHAIN, VERIFY_CRL_CHECK_LEAF, VERIFY_DEFAULT, VERIFY_X509_STRICT + from _ssl import HAS_SNI, HAS_ECDH, HAS_NPN, HAS_ALPN + from _ssl import _OPENSSL_API_VERSION + from _ssl import PROTOCOL_SSLv23, PROTOCOL_TLSv1, PROTOCOL_TLSv1_1, PROTOCOL_TLSv1_2 + from _ssl import PROTOCOL_TLS, PROTOCOL_TLS_CLIENT, PROTOCOL_TLS_SERVER + """ + ) + + +register_module_extender(MANAGER, "ssl", ssl_transform) diff --git a/WebCrawler/venv/Lib/site-packages/astroid/brain/brain_subprocess.py b/WebCrawler/venv/Lib/site-packages/astroid/brain/brain_subprocess.py new file mode 100644 index 0000000..ab7d5d7 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/astroid/brain/brain_subprocess.py @@ -0,0 +1,146 @@ +# Copyright (c) 2016-2020 Claudiu Popa +# Copyright (c) 2017 Hugo +# Copyright (c) 2018 Peter Talley +# Copyright (c) 2018 Bryce Guinta +# Copyright (c) 2019 Hugo van Kemenade + +# Licensed under the LGPL: https://www.gnu.org/licenses/old-licenses/lgpl-2.1.en.html +# For details: https://github.com/PyCQA/astroid/blob/master/COPYING.LESSER + +import sys +import textwrap + +import astroid + + +PY37 = sys.version_info >= (3, 7) +PY36 = sys.version_info >= (3, 6) + + +def _subprocess_transform(): + communicate = (bytes("string", "ascii"), bytes("string", "ascii")) + communicate_signature = "def communicate(self, input=None, timeout=None)" + if PY37: + init = """ + def __init__(self, args, bufsize=0, executable=None, + stdin=None, stdout=None, stderr=None, + preexec_fn=None, close_fds=False, shell=False, + cwd=None, env=None, universal_newlines=False, + startupinfo=None, creationflags=0, restore_signals=True, + start_new_session=False, pass_fds=(), *, + encoding=None, errors=None, text=None): + pass + """ + elif PY36: + init = """ + def __init__(self, args, bufsize=0, executable=None, + stdin=None, stdout=None, stderr=None, + preexec_fn=None, close_fds=False, shell=False, + cwd=None, env=None, universal_newlines=False, + startupinfo=None, creationflags=0, restore_signals=True, + start_new_session=False, pass_fds=(), *, + encoding=None, errors=None): + pass + """ + else: + init = """ + def __init__(self, args, bufsize=0, executable=None, + stdin=None, stdout=None, stderr=None, + preexec_fn=None, close_fds=False, shell=False, + cwd=None, env=None, universal_newlines=False, + startupinfo=None, creationflags=0, restore_signals=True, + start_new_session=False, pass_fds=()): + pass + """ + wait_signature = "def wait(self, timeout=None)" + ctx_manager = """ + def __enter__(self): return self + def __exit__(self, *args): pass + """ + py3_args = "args = []" + + if PY37: + check_output_signature = """ + check_output( + args, *, + stdin=None, + stderr=None, + shell=False, + cwd=None, + encoding=None, + errors=None, + universal_newlines=False, + timeout=None, + env=None, + text=None, + restore_signals=True, + preexec_fn=None, + pass_fds=(), + input=None, + start_new_session=False + ): + """.strip() + else: + check_output_signature = """ + check_output( + args, *, + stdin=None, + stderr=None, + shell=False, + cwd=None, + encoding=None, + errors=None, + universal_newlines=False, + timeout=None, + env=None, + restore_signals=True, + preexec_fn=None, + pass_fds=(), + input=None, + start_new_session=False + ): + """.strip() + + code = textwrap.dedent( + """ + def %(check_output_signature)s + if universal_newlines: + return "" + return b"" + + class Popen(object): + returncode = pid = 0 + stdin = stdout = stderr = file() + %(py3_args)s + + %(communicate_signature)s: + return %(communicate)r + %(wait_signature)s: + return self.returncode + def poll(self): + return self.returncode + def send_signal(self, signal): + pass + def terminate(self): + pass + def kill(self): + pass + %(ctx_manager)s + """ + % { + "check_output_signature": check_output_signature, + "communicate": communicate, + "communicate_signature": communicate_signature, + "wait_signature": wait_signature, + "ctx_manager": ctx_manager, + "py3_args": py3_args, + } + ) + + init_lines = textwrap.dedent(init).splitlines() + indented_init = "\n".join(" " * 4 + line for line in init_lines) + code += indented_init + return astroid.parse(code) + + +astroid.register_module_extender(astroid.MANAGER, "subprocess", _subprocess_transform) diff --git a/WebCrawler/venv/Lib/site-packages/astroid/brain/brain_threading.py b/WebCrawler/venv/Lib/site-packages/astroid/brain/brain_threading.py new file mode 100644 index 0000000..ba3085b --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/astroid/brain/brain_threading.py @@ -0,0 +1,31 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2016, 2018-2019 Claudiu Popa +# Copyright (c) 2017 Łukasz Rogalski + +# Licensed under the LGPL: https://www.gnu.org/licenses/old-licenses/lgpl-2.1.en.html +# For details: https://github.com/PyCQA/astroid/blob/master/COPYING.LESSER +import astroid + + +def _thread_transform(): + return astroid.parse( + """ + class lock(object): + def acquire(self, blocking=True, timeout=-1): + return False + def release(self): + pass + def __enter__(self): + return True + def __exit__(self, *args): + pass + def locked(self): + return False + + def Lock(): + return lock() + """ + ) + + +astroid.register_module_extender(astroid.MANAGER, "threading", _thread_transform) diff --git a/WebCrawler/venv/Lib/site-packages/astroid/brain/brain_typing.py b/WebCrawler/venv/Lib/site-packages/astroid/brain/brain_typing.py new file mode 100644 index 0000000..9ff7227 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/astroid/brain/brain_typing.py @@ -0,0 +1,96 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2017-2018 Claudiu Popa +# Copyright (c) 2017 Łukasz Rogalski +# Copyright (c) 2017 David Euresti +# Copyright (c) 2018 Bryce Guinta + +"""Astroid hooks for typing.py support.""" +import typing + +from astroid import ( + MANAGER, + UseInferenceDefault, + extract_node, + inference_tip, + nodes, + InferenceError, +) + + +TYPING_NAMEDTUPLE_BASENAMES = {"NamedTuple", "typing.NamedTuple"} +TYPING_TYPEVARS = {"TypeVar", "NewType"} +TYPING_TYPEVARS_QUALIFIED = {"typing.TypeVar", "typing.NewType"} +TYPING_TYPE_TEMPLATE = """ +class Meta(type): + def __getitem__(self, item): + return self + + @property + def __args__(self): + return () + +class {0}(metaclass=Meta): + pass +""" +TYPING_MEMBERS = set(typing.__all__) + + +def looks_like_typing_typevar_or_newtype(node): + func = node.func + if isinstance(func, nodes.Attribute): + return func.attrname in TYPING_TYPEVARS + if isinstance(func, nodes.Name): + return func.name in TYPING_TYPEVARS + return False + + +def infer_typing_typevar_or_newtype(node, context=None): + """Infer a typing.TypeVar(...) or typing.NewType(...) call""" + try: + func = next(node.func.infer(context=context)) + except InferenceError as exc: + raise UseInferenceDefault from exc + + if func.qname() not in TYPING_TYPEVARS_QUALIFIED: + raise UseInferenceDefault + if not node.args: + raise UseInferenceDefault + + typename = node.args[0].as_string().strip("'") + node = extract_node(TYPING_TYPE_TEMPLATE.format(typename)) + return node.infer(context=context) + + +def _looks_like_typing_subscript(node): + """Try to figure out if a Subscript node *might* be a typing-related subscript""" + if isinstance(node, nodes.Name): + return node.name in TYPING_MEMBERS + elif isinstance(node, nodes.Attribute): + return node.attrname in TYPING_MEMBERS + elif isinstance(node, nodes.Subscript): + return _looks_like_typing_subscript(node.value) + return False + + +def infer_typing_attr(node, context=None): + """Infer a typing.X[...] subscript""" + try: + value = next(node.value.infer()) + except InferenceError as exc: + raise UseInferenceDefault from exc + + if not value.qname().startswith("typing."): + raise UseInferenceDefault + + node = extract_node(TYPING_TYPE_TEMPLATE.format(value.qname().split(".")[-1])) + return node.infer(context=context) + + +MANAGER.register_transform( + nodes.Call, + inference_tip(infer_typing_typevar_or_newtype), + looks_like_typing_typevar_or_newtype, +) +MANAGER.register_transform( + nodes.Subscript, inference_tip(infer_typing_attr), _looks_like_typing_subscript +) diff --git a/WebCrawler/venv/Lib/site-packages/astroid/brain/brain_uuid.py b/WebCrawler/venv/Lib/site-packages/astroid/brain/brain_uuid.py new file mode 100644 index 0000000..5a33fc2 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/astroid/brain/brain_uuid.py @@ -0,0 +1,20 @@ +# Copyright (c) 2017-2018 Claudiu Popa + +# Licensed under the LGPL: https://www.gnu.org/licenses/old-licenses/lgpl-2.1.en.html +# For details: https://github.com/PyCQA/astroid/blob/master/COPYING.LESSER + +"""Astroid hooks for the UUID module.""" + + +from astroid import MANAGER +from astroid import nodes + + +def _patch_uuid_class(node): + # The .int member is patched using __dict__ + node.locals["int"] = [nodes.Const(0, parent=node)] + + +MANAGER.register_transform( + nodes.ClassDef, _patch_uuid_class, lambda node: node.qname() == "uuid.UUID" +) diff --git a/WebCrawler/venv/Lib/site-packages/astroid/builder.py b/WebCrawler/venv/Lib/site-packages/astroid/builder.py new file mode 100644 index 0000000..142764b --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/astroid/builder.py @@ -0,0 +1,455 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2006-2011, 2013-2014 LOGILAB S.A. (Paris, FRANCE) +# Copyright (c) 2013 Phil Schaf +# Copyright (c) 2014-2019 Claudiu Popa +# Copyright (c) 2014-2015 Google, Inc. +# Copyright (c) 2014 Alexander Presnyakov +# Copyright (c) 2015-2016 Ceridwen +# Copyright (c) 2016 Derek Gustafson +# Copyright (c) 2017 Łukasz Rogalski +# Copyright (c) 2018 Anthony Sottile + +# Licensed under the LGPL: https://www.gnu.org/licenses/old-licenses/lgpl-2.1.en.html +# For details: https://github.com/PyCQA/astroid/blob/master/COPYING.LESSER + +"""The AstroidBuilder makes astroid from living object and / or from _ast + +The builder is not thread safe and can't be used to parse different sources +at the same time. +""" + +import os +import textwrap +from tokenize import detect_encoding + +from astroid._ast import get_parser_module +from astroid import bases +from astroid import exceptions +from astroid import manager +from astroid import modutils +from astroid import raw_building +from astroid import rebuilder +from astroid import nodes +from astroid import util + +objects = util.lazy_import("objects") + +# The name of the transient function that is used to +# wrap expressions to be extracted when calling +# extract_node. +_TRANSIENT_FUNCTION = "__" + +# The comment used to select a statement to be extracted +# when calling extract_node. +_STATEMENT_SELECTOR = "#@" +MISPLACED_TYPE_ANNOTATION_ERROR = "misplaced type annotation" +MANAGER = manager.AstroidManager() + + +def open_source_file(filename): + with open(filename, "rb") as byte_stream: + encoding = detect_encoding(byte_stream.readline)[0] + stream = open(filename, "r", newline=None, encoding=encoding) + data = stream.read() + return stream, encoding, data + + +def _can_assign_attr(node, attrname): + try: + slots = node.slots() + except NotImplementedError: + pass + else: + if slots and attrname not in {slot.value for slot in slots}: + return False + return True + + +class AstroidBuilder(raw_building.InspectBuilder): + """Class for building an astroid tree from source code or from a live module. + + The param *manager* specifies the manager class which should be used. + If no manager is given, then the default one will be used. The + param *apply_transforms* determines if the transforms should be + applied after the tree was built from source or from a live object, + by default being True. + """ + + # pylint: disable=redefined-outer-name + def __init__(self, manager=None, apply_transforms=True): + super().__init__() + self._manager = manager or MANAGER + self._apply_transforms = apply_transforms + + def module_build(self, module, modname=None): + """Build an astroid from a living module instance.""" + node = None + path = getattr(module, "__file__", None) + if path is not None: + path_, ext = os.path.splitext(modutils._path_from_filename(path)) + if ext in (".py", ".pyc", ".pyo") and os.path.exists(path_ + ".py"): + node = self.file_build(path_ + ".py", modname) + if node is None: + # this is a built-in module + # get a partial representation by introspection + node = self.inspect_build(module, modname=modname, path=path) + if self._apply_transforms: + # We have to handle transformation by ourselves since the + # rebuilder isn't called for builtin nodes + node = self._manager.visit_transforms(node) + return node + + def file_build(self, path, modname=None): + """Build astroid from a source code file (i.e. from an ast) + + *path* is expected to be a python source file + """ + try: + stream, encoding, data = open_source_file(path) + except IOError as exc: + raise exceptions.AstroidBuildingError( + "Unable to load file {path}:\n{error}", + modname=modname, + path=path, + error=exc, + ) from exc + except (SyntaxError, LookupError) as exc: + raise exceptions.AstroidSyntaxError( + "Python 3 encoding specification error or unknown encoding:\n" + "{error}", + modname=modname, + path=path, + error=exc, + ) from exc + except UnicodeError as exc: # wrong encoding + # detect_encoding returns utf-8 if no encoding specified + raise exceptions.AstroidBuildingError( + "Wrong or no encoding specified for {filename}.", filename=path + ) from exc + with stream: + # get module name if necessary + if modname is None: + try: + modname = ".".join(modutils.modpath_from_file(path)) + except ImportError: + modname = os.path.splitext(os.path.basename(path))[0] + # build astroid representation + module = self._data_build(data, modname, path) + return self._post_build(module, encoding) + + def string_build(self, data, modname="", path=None): + """Build astroid from source code string.""" + module = self._data_build(data, modname, path) + module.file_bytes = data.encode("utf-8") + return self._post_build(module, "utf-8") + + def _post_build(self, module, encoding): + """Handles encoding and delayed nodes after a module has been built""" + module.file_encoding = encoding + self._manager.cache_module(module) + # post tree building steps after we stored the module in the cache: + for from_node in module._import_from_nodes: + if from_node.modname == "__future__": + for symbol, _ in from_node.names: + module.future_imports.add(symbol) + self.add_from_names_to_locals(from_node) + # handle delayed assattr nodes + for delayed in module._delayed_assattr: + self.delayed_assattr(delayed) + + # Visit the transforms + if self._apply_transforms: + module = self._manager.visit_transforms(module) + return module + + def _data_build(self, data, modname, path): + """Build tree node from data and add some informations""" + try: + node, parser_module = _parse_string(data, type_comments=True) + except (TypeError, ValueError, SyntaxError) as exc: + raise exceptions.AstroidSyntaxError( + "Parsing Python code failed:\n{error}", + source=data, + modname=modname, + path=path, + error=exc, + ) from exc + + if path is not None: + node_file = os.path.abspath(path) + else: + node_file = "" + if modname.endswith(".__init__"): + modname = modname[:-9] + package = True + else: + package = ( + path is not None + and os.path.splitext(os.path.basename(path))[0] == "__init__" + ) + builder = rebuilder.TreeRebuilder(self._manager, parser_module) + module = builder.visit_module(node, modname, node_file, package) + module._import_from_nodes = builder._import_from_nodes + module._delayed_assattr = builder._delayed_assattr + return module + + def add_from_names_to_locals(self, node): + """Store imported names to the locals + + Resort the locals if coming from a delayed node + """ + _key_func = lambda node: node.fromlineno + + def sort_locals(my_list): + my_list.sort(key=_key_func) + + for (name, asname) in node.names: + if name == "*": + try: + imported = node.do_import_module() + except exceptions.AstroidBuildingError: + continue + for name in imported.public_names(): + node.parent.set_local(name, node) + sort_locals(node.parent.scope().locals[name]) + else: + node.parent.set_local(asname or name, node) + sort_locals(node.parent.scope().locals[asname or name]) + + def delayed_assattr(self, node): + """Visit a AssAttr node + + This adds name to locals and handle members definition. + """ + try: + frame = node.frame() + for inferred in node.expr.infer(): + if inferred is util.Uninferable: + continue + try: + cls = inferred.__class__ + if cls is bases.Instance or cls is objects.ExceptionInstance: + inferred = inferred._proxied + iattrs = inferred.instance_attrs + if not _can_assign_attr(inferred, node.attrname): + continue + elif isinstance(inferred, bases.Instance): + # Const, Tuple or other containers that inherit from + # `Instance` + continue + elif inferred.is_function: + iattrs = inferred.instance_attrs + else: + iattrs = inferred.locals + except AttributeError: + # XXX log error + continue + values = iattrs.setdefault(node.attrname, []) + if node in values: + continue + # get assign in __init__ first XXX useful ? + if ( + frame.name == "__init__" + and values + and values[0].frame().name != "__init__" + ): + values.insert(0, node) + else: + values.append(node) + except exceptions.InferenceError: + pass + + +def build_namespace_package_module(name, path): + return nodes.Module(name, doc="", path=path, package=True) + + +def parse(code, module_name="", path=None, apply_transforms=True): + """Parses a source string in order to obtain an astroid AST from it + + :param str code: The code for the module. + :param str module_name: The name for the module, if any + :param str path: The path for the module + :param bool apply_transforms: + Apply the transforms for the give code. Use it if you + don't want the default transforms to be applied. + """ + code = textwrap.dedent(code) + builder = AstroidBuilder(manager=MANAGER, apply_transforms=apply_transforms) + return builder.string_build(code, modname=module_name, path=path) + + +def _extract_expressions(node): + """Find expressions in a call to _TRANSIENT_FUNCTION and extract them. + + The function walks the AST recursively to search for expressions that + are wrapped into a call to _TRANSIENT_FUNCTION. If it finds such an + expression, it completely removes the function call node from the tree, + replacing it by the wrapped expression inside the parent. + + :param node: An astroid node. + :type node: astroid.bases.NodeNG + :yields: The sequence of wrapped expressions on the modified tree + expression can be found. + """ + if ( + isinstance(node, nodes.Call) + and isinstance(node.func, nodes.Name) + and node.func.name == _TRANSIENT_FUNCTION + ): + real_expr = node.args[0] + real_expr.parent = node.parent + # Search for node in all _astng_fields (the fields checked when + # get_children is called) of its parent. Some of those fields may + # be lists or tuples, in which case the elements need to be checked. + # When we find it, replace it by real_expr, so that the AST looks + # like no call to _TRANSIENT_FUNCTION ever took place. + for name in node.parent._astroid_fields: + child = getattr(node.parent, name) + if isinstance(child, (list, tuple)): + for idx, compound_child in enumerate(child): + if compound_child is node: + child[idx] = real_expr + elif child is node: + setattr(node.parent, name, real_expr) + yield real_expr + else: + for child in node.get_children(): + yield from _extract_expressions(child) + + +def _find_statement_by_line(node, line): + """Extracts the statement on a specific line from an AST. + + If the line number of node matches line, it will be returned; + otherwise its children are iterated and the function is called + recursively. + + :param node: An astroid node. + :type node: astroid.bases.NodeNG + :param line: The line number of the statement to extract. + :type line: int + :returns: The statement on the line, or None if no statement for the line + can be found. + :rtype: astroid.bases.NodeNG or None + """ + if isinstance(node, (nodes.ClassDef, nodes.FunctionDef)): + # This is an inaccuracy in the AST: the nodes that can be + # decorated do not carry explicit information on which line + # the actual definition (class/def), but .fromline seems to + # be close enough. + node_line = node.fromlineno + else: + node_line = node.lineno + + if node_line == line: + return node + + for child in node.get_children(): + result = _find_statement_by_line(child, line) + if result: + return result + + return None + + +def extract_node(code, module_name=""): + """Parses some Python code as a module and extracts a designated AST node. + + Statements: + To extract one or more statement nodes, append #@ to the end of the line + + Examples: + >>> def x(): + >>> def y(): + >>> return 1 #@ + + The return statement will be extracted. + + >>> class X(object): + >>> def meth(self): #@ + >>> pass + + The function object 'meth' will be extracted. + + Expressions: + To extract arbitrary expressions, surround them with the fake + function call __(...). After parsing, the surrounded expression + will be returned and the whole AST (accessible via the returned + node's parent attribute) will look like the function call was + never there in the first place. + + Examples: + >>> a = __(1) + + The const node will be extracted. + + >>> def x(d=__(foo.bar)): pass + + The node containing the default argument will be extracted. + + >>> def foo(a, b): + >>> return 0 < __(len(a)) < b + + The node containing the function call 'len' will be extracted. + + If no statements or expressions are selected, the last toplevel + statement will be returned. + + If the selected statement is a discard statement, (i.e. an expression + turned into a statement), the wrapped expression is returned instead. + + For convenience, singleton lists are unpacked. + + :param str code: A piece of Python code that is parsed as + a module. Will be passed through textwrap.dedent first. + :param str module_name: The name of the module. + :returns: The designated node from the parse tree, or a list of nodes. + :rtype: astroid.bases.NodeNG, or a list of nodes. + """ + + def _extract(node): + if isinstance(node, nodes.Expr): + return node.value + + return node + + requested_lines = [] + for idx, line in enumerate(code.splitlines()): + if line.strip().endswith(_STATEMENT_SELECTOR): + requested_lines.append(idx + 1) + + tree = parse(code, module_name=module_name) + if not tree.body: + raise ValueError("Empty tree, cannot extract from it") + + extracted = [] + if requested_lines: + extracted = [_find_statement_by_line(tree, line) for line in requested_lines] + + # Modifies the tree. + extracted.extend(_extract_expressions(tree)) + + if not extracted: + extracted.append(tree.body[-1]) + + extracted = [_extract(node) for node in extracted] + if len(extracted) == 1: + return extracted[0] + return extracted + + +def _parse_string(data, type_comments=True): + parser_module = get_parser_module(type_comments=type_comments) + try: + parsed = parser_module.parse(data + "\n", type_comments=type_comments) + except SyntaxError as exc: + # If the type annotations are misplaced for some reason, we do not want + # to fail the entire parsing of the file, so we need to retry the parsing without + # type comment support. + if exc.args[0] != MISPLACED_TYPE_ANNOTATION_ERROR or not type_comments: + raise + + parser_module = get_parser_module(type_comments=False) + parsed = parser_module.parse(data + "\n", type_comments=False) + return parsed, parser_module diff --git a/WebCrawler/venv/Lib/site-packages/astroid/context.py b/WebCrawler/venv/Lib/site-packages/astroid/context.py new file mode 100644 index 0000000..40cebf2 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/astroid/context.py @@ -0,0 +1,179 @@ +# Copyright (c) 2015-2016, 2018-2019 Claudiu Popa +# Copyright (c) 2015-2016 Ceridwen +# Copyright (c) 2018 Bryce Guinta +# Copyright (c) 2018 Nick Drozd + +# Licensed under the LGPL: https://www.gnu.org/licenses/old-licenses/lgpl-2.1.en.html +# For details: https://github.com/PyCQA/astroid/blob/master/COPYING.LESSER + +"""Various context related utilities, including inference and call contexts.""" +import contextlib +import pprint +from typing import Optional + + +class InferenceContext: + """Provide context for inference + + Store already inferred nodes to save time + Account for already visited nodes to infinite stop infinite recursion + """ + + __slots__ = ( + "path", + "lookupname", + "callcontext", + "boundnode", + "inferred", + "extra_context", + ) + + def __init__(self, path=None, inferred=None): + self.path = path or set() + """ + :type: set(tuple(NodeNG, optional(str))) + + Path of visited nodes and their lookupname + + Currently this key is ``(node, context.lookupname)`` + """ + self.lookupname = None + """ + :type: optional[str] + + The original name of the node + + e.g. + foo = 1 + The inference of 'foo' is nodes.Const(1) but the lookup name is 'foo' + """ + self.callcontext = None + """ + :type: optional[CallContext] + + The call arguments and keywords for the given context + """ + self.boundnode = None + """ + :type: optional[NodeNG] + + The bound node of the given context + + e.g. the bound node of object.__new__(cls) is the object node + """ + self.inferred = inferred or {} + """ + :type: dict(seq, seq) + + Inferred node contexts to their mapped results + Currently the key is ``(node, lookupname, callcontext, boundnode)`` + and the value is tuple of the inferred results + """ + self.extra_context = {} + """ + :type: dict(NodeNG, Context) + + Context that needs to be passed down through call stacks + for call arguments + """ + + def push(self, node): + """Push node into inference path + + :return: True if node is already in context path else False + :rtype: bool + + Allows one to see if the given node has already + been looked at for this inference context""" + name = self.lookupname + if (node, name) in self.path: + return True + + self.path.add((node, name)) + return False + + def clone(self): + """Clone inference path + + For example, each side of a binary operation (BinOp) + starts with the same context but diverge as each side is inferred + so the InferenceContext will need be cloned""" + # XXX copy lookupname/callcontext ? + clone = InferenceContext(self.path, inferred=self.inferred) + clone.callcontext = self.callcontext + clone.boundnode = self.boundnode + clone.extra_context = self.extra_context + return clone + + def cache_generator(self, key, generator): + """Cache result of generator into dictionary + + Used to cache inference results""" + results = [] + for result in generator: + results.append(result) + yield result + + self.inferred[key] = tuple(results) + + @contextlib.contextmanager + def restore_path(self): + path = set(self.path) + yield + self.path = path + + def __str__(self): + state = ( + "%s=%s" + % (field, pprint.pformat(getattr(self, field), width=80 - len(field))) + for field in self.__slots__ + ) + return "%s(%s)" % (type(self).__name__, ",\n ".join(state)) + + +class CallContext: + """Holds information for a call site.""" + + __slots__ = ("args", "keywords") + + def __init__(self, args, keywords=None): + """ + :param List[NodeNG] args: Call positional arguments + :param Union[List[nodes.Keyword], None] keywords: Call keywords + """ + self.args = args + if keywords: + keywords = [(arg.arg, arg.value) for arg in keywords] + else: + keywords = [] + self.keywords = keywords + + +def copy_context(context: Optional[InferenceContext]) -> InferenceContext: + """Clone a context if given, or return a fresh contexxt""" + if context is not None: + return context.clone() + + return InferenceContext() + + +def bind_context_to_node(context, node): + """Give a context a boundnode + to retrieve the correct function name or attribute value + with from further inference. + + Do not use an existing context since the boundnode could then + be incorrectly propagated higher up in the call stack. + + :param context: Context to use + :type context: Optional(context) + + :param node: Node to do name lookups from + :type node NodeNG: + + :returns: A new context + :rtype: InferenceContext + """ + context = copy_context(context) + context.boundnode = node + return context diff --git a/WebCrawler/venv/Lib/site-packages/astroid/decorators.py b/WebCrawler/venv/Lib/site-packages/astroid/decorators.py new file mode 100644 index 0000000..0f3632c --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/astroid/decorators.py @@ -0,0 +1,142 @@ +# Copyright (c) 2015-2016, 2018 Claudiu Popa +# Copyright (c) 2015-2016 Ceridwen +# Copyright (c) 2015 Florian Bruhin +# Copyright (c) 2016 Derek Gustafson +# Copyright (c) 2018 Nick Drozd +# Copyright (c) 2018 Tomas Gavenciak +# Copyright (c) 2018 Ashley Whetter +# Copyright (c) 2018 HoverHell +# Copyright (c) 2018 Bryce Guinta + +# Licensed under the LGPL: https://www.gnu.org/licenses/old-licenses/lgpl-2.1.en.html +# For details: https://github.com/PyCQA/astroid/blob/master/COPYING.LESSER + +""" A few useful function/method decorators.""" + +import functools + +import wrapt + +from astroid import context as contextmod +from astroid import exceptions +from astroid import util + + +@wrapt.decorator +def cached(func, instance, args, kwargs): + """Simple decorator to cache result of method calls without args.""" + cache = getattr(instance, "__cache", None) + if cache is None: + instance.__cache = cache = {} + try: + return cache[func] + except KeyError: + cache[func] = result = func(*args, **kwargs) + return result + + +class cachedproperty: + """ Provides a cached property equivalent to the stacking of + @cached and @property, but more efficient. + + After first usage, the becomes part of the object's + __dict__. Doing: + + del obj. empties the cache. + + Idea taken from the pyramid_ framework and the mercurial_ project. + + .. _pyramid: http://pypi.python.org/pypi/pyramid + .. _mercurial: http://pypi.python.org/pypi/Mercurial + """ + + __slots__ = ("wrapped",) + + def __init__(self, wrapped): + try: + wrapped.__name__ + except AttributeError as exc: + raise TypeError("%s must have a __name__ attribute" % wrapped) from exc + self.wrapped = wrapped + + @property + def __doc__(self): + doc = getattr(self.wrapped, "__doc__", None) + return "%s" % ( + "\n%s" % doc if doc else "" + ) + + def __get__(self, inst, objtype=None): + if inst is None: + return self + val = self.wrapped(inst) + setattr(inst, self.wrapped.__name__, val) + return val + + +def path_wrapper(func): + """return the given infer function wrapped to handle the path + + Used to stop inference if the node has already been looked + at for a given `InferenceContext` to prevent infinite recursion + """ + + @functools.wraps(func) + def wrapped(node, context=None, _func=func, **kwargs): + """wrapper function handling context""" + if context is None: + context = contextmod.InferenceContext() + if context.push(node): + return None + + yielded = set() + generator = _func(node, context, **kwargs) + try: + while True: + res = next(generator) + # unproxy only true instance, not const, tuple, dict... + if res.__class__.__name__ == "Instance": + ares = res._proxied + else: + ares = res + if ares not in yielded: + yield res + yielded.add(ares) + except StopIteration as error: + if error.args: + return error.args[0] + return None + + return wrapped + + +@wrapt.decorator +def yes_if_nothing_inferred(func, instance, args, kwargs): + generator = func(*args, **kwargs) + + try: + yield next(generator) + except StopIteration: + # generator is empty + yield util.Uninferable + return + + yield from generator + + +@wrapt.decorator +def raise_if_nothing_inferred(func, instance, args, kwargs): + generator = func(*args, **kwargs) + + try: + yield next(generator) + except StopIteration as error: + # generator is empty + if error.args: + # pylint: disable=not-a-mapping + raise exceptions.InferenceError(**error.args[0]) + raise exceptions.InferenceError( + "StopIteration raised without any error information." + ) + + yield from generator diff --git a/WebCrawler/venv/Lib/site-packages/astroid/exceptions.py b/WebCrawler/venv/Lib/site-packages/astroid/exceptions.py new file mode 100644 index 0000000..08e72c1 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/astroid/exceptions.py @@ -0,0 +1,230 @@ +# Copyright (c) 2007, 2009-2010, 2013 LOGILAB S.A. (Paris, FRANCE) +# Copyright (c) 2014 Google, Inc. +# Copyright (c) 2015-2018 Claudiu Popa +# Copyright (c) 2015-2016 Ceridwen +# Copyright (c) 2016 Derek Gustafson +# Copyright (c) 2018 Bryce Guinta + +# Licensed under the LGPL: https://www.gnu.org/licenses/old-licenses/lgpl-2.1.en.html +# For details: https://github.com/PyCQA/astroid/blob/master/COPYING.LESSER + +"""this module contains exceptions used in the astroid library +""" +from astroid import util + + +class AstroidError(Exception): + """base exception class for all astroid related exceptions + + AstroidError and its subclasses are structured, intended to hold + objects representing state when the exception is thrown. Field + values are passed to the constructor as keyword-only arguments. + Each subclass has its own set of standard fields, but use your + best judgment to decide whether a specific exception instance + needs more or fewer fields for debugging. Field values may be + used to lazily generate the error message: self.message.format() + will be called with the field names and values supplied as keyword + arguments. + """ + + def __init__(self, message="", **kws): + super().__init__(message) + self.message = message + for key, value in kws.items(): + setattr(self, key, value) + + def __str__(self): + return self.message.format(**vars(self)) + + +class AstroidBuildingError(AstroidError): + """exception class when we are unable to build an astroid representation + + Standard attributes: + modname: Name of the module that AST construction failed for. + error: Exception raised during construction. + """ + + def __init__(self, message="Failed to import module {modname}.", **kws): + super().__init__(message, **kws) + + +class AstroidImportError(AstroidBuildingError): + """Exception class used when a module can't be imported by astroid.""" + + +class TooManyLevelsError(AstroidImportError): + """Exception class which is raised when a relative import was beyond the top-level. + + Standard attributes: + level: The level which was attempted. + name: the name of the module on which the relative import was attempted. + """ + + level = None + name = None + + def __init__( + self, + message="Relative import with too many levels " "({level}) for module {name!r}", + **kws + ): + super().__init__(message, **kws) + + +class AstroidSyntaxError(AstroidBuildingError): + """Exception class used when a module can't be parsed.""" + + +class NoDefault(AstroidError): + """raised by function's `default_value` method when an argument has + no default value + + Standard attributes: + func: Function node. + name: Name of argument without a default. + """ + + func = None + name = None + + def __init__(self, message="{func!r} has no default for {name!r}.", **kws): + super().__init__(message, **kws) + + +class ResolveError(AstroidError): + """Base class of astroid resolution/inference error. + + ResolveError is not intended to be raised. + + Standard attributes: + context: InferenceContext object. + """ + + context = None + + +class MroError(ResolveError): + """Error raised when there is a problem with method resolution of a class. + + Standard attributes: + mros: A sequence of sequences containing ClassDef nodes. + cls: ClassDef node whose MRO resolution failed. + context: InferenceContext object. + """ + + mros = () + cls = None + + def __str__(self): + mro_names = ", ".join( + "({})".format(", ".join(b.name for b in m)) for m in self.mros + ) + return self.message.format(mros=mro_names, cls=self.cls) + + +class DuplicateBasesError(MroError): + """Error raised when there are duplicate bases in the same class bases.""" + + +class InconsistentMroError(MroError): + """Error raised when a class's MRO is inconsistent.""" + + +class SuperError(ResolveError): + """Error raised when there is a problem with a *super* call. + + Standard attributes: + *super_*: The Super instance that raised the exception. + context: InferenceContext object. + """ + + super_ = None + + def __str__(self): + return self.message.format(**vars(self.super_)) + + +class InferenceError(ResolveError): + """raised when we are unable to infer a node + + Standard attributes: + node: The node inference was called on. + context: InferenceContext object. + """ + + node = None + context = None + + def __init__(self, message="Inference failed for {node!r}.", **kws): + super().__init__(message, **kws) + + +# Why does this inherit from InferenceError rather than ResolveError? +# Changing it causes some inference tests to fail. +class NameInferenceError(InferenceError): + """Raised when a name lookup fails, corresponds to NameError. + + Standard attributes: + name: The name for which lookup failed, as a string. + scope: The node representing the scope in which the lookup occurred. + context: InferenceContext object. + """ + + name = None + scope = None + + def __init__(self, message="{name!r} not found in {scope!r}.", **kws): + super().__init__(message, **kws) + + +class AttributeInferenceError(ResolveError): + """Raised when an attribute lookup fails, corresponds to AttributeError. + + Standard attributes: + target: The node for which lookup failed. + attribute: The attribute for which lookup failed, as a string. + context: InferenceContext object. + """ + + target = None + attribute = None + + def __init__(self, message="{attribute!r} not found on {target!r}.", **kws): + super().__init__(message, **kws) + + +class UseInferenceDefault(Exception): + """exception to be raised in custom inference function to indicate that it + should go back to the default behaviour + """ + + +class _NonDeducibleTypeHierarchy(Exception): + """Raised when is_subtype / is_supertype can't deduce the relation between two types.""" + + +class AstroidIndexError(AstroidError): + """Raised when an Indexable / Mapping does not have an index / key.""" + + +class AstroidTypeError(AstroidError): + """Raised when a TypeError would be expected in Python code.""" + + +class InferenceOverwriteError(AstroidError): + """Raised when an inference tip is overwritten + + Currently only used for debugging. + """ + + +# Backwards-compatibility aliases +OperationError = util.BadOperationMessage +UnaryOperationError = util.BadUnaryOperationMessage +BinaryOperationError = util.BadBinaryOperationMessage + +SuperArgumentTypeError = SuperError +UnresolvableName = NameInferenceError +NotFoundError = AttributeInferenceError +AstroidBuildingException = AstroidBuildingError diff --git a/WebCrawler/venv/Lib/site-packages/astroid/helpers.py b/WebCrawler/venv/Lib/site-packages/astroid/helpers.py new file mode 100644 index 0000000..1c84651 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/astroid/helpers.py @@ -0,0 +1,282 @@ +# Copyright (c) 2015-2020 Claudiu Popa +# Copyright (c) 2015-2016 Ceridwen +# Copyright (c) 2018 Bryce Guinta + +# Licensed under the LGPL: https://www.gnu.org/licenses/old-licenses/lgpl-2.1.en.html +# For details: https://github.com/PyCQA/astroid/blob/master/COPYING.LESSER + + +""" +Various helper utilities. +""" + +import builtins as builtins_mod + +from astroid import bases +from astroid import context as contextmod +from astroid import exceptions +from astroid import manager +from astroid import nodes +from astroid import raw_building +from astroid import scoped_nodes +from astroid import util + + +BUILTINS = builtins_mod.__name__ + + +def _build_proxy_class(cls_name, builtins): + proxy = raw_building.build_class(cls_name) + proxy.parent = builtins + return proxy + + +def _function_type(function, builtins): + if isinstance(function, scoped_nodes.Lambda): + if function.root().name == BUILTINS: + cls_name = "builtin_function_or_method" + else: + cls_name = "function" + elif isinstance(function, bases.BoundMethod): + cls_name = "method" + elif isinstance(function, bases.UnboundMethod): + cls_name = "function" + return _build_proxy_class(cls_name, builtins) + + +def _object_type(node, context=None): + astroid_manager = manager.AstroidManager() + builtins = astroid_manager.builtins_module + context = context or contextmod.InferenceContext() + + for inferred in node.infer(context=context): + if isinstance(inferred, scoped_nodes.ClassDef): + if inferred.newstyle: + metaclass = inferred.metaclass(context=context) + if metaclass: + yield metaclass + continue + yield builtins.getattr("type")[0] + elif isinstance(inferred, (scoped_nodes.Lambda, bases.UnboundMethod)): + yield _function_type(inferred, builtins) + elif isinstance(inferred, scoped_nodes.Module): + yield _build_proxy_class("module", builtins) + else: + yield inferred._proxied + + +def object_type(node, context=None): + """Obtain the type of the given node + + This is used to implement the ``type`` builtin, which means that it's + used for inferring type calls, as well as used in a couple of other places + in the inference. + The node will be inferred first, so this function can support all + sorts of objects, as long as they support inference. + """ + + try: + types = set(_object_type(node, context)) + except exceptions.InferenceError: + return util.Uninferable + if len(types) > 1 or not types: + return util.Uninferable + return list(types)[0] + + +def _object_type_is_subclass(obj_type, class_or_seq, context=None): + if not isinstance(class_or_seq, (tuple, list)): + class_seq = (class_or_seq,) + else: + class_seq = class_or_seq + + if obj_type is util.Uninferable: + return util.Uninferable + + # Instances are not types + class_seq = [ + item if not isinstance(item, bases.Instance) else util.Uninferable + for item in class_seq + ] + # strict compatibility with issubclass + # issubclass(type, (object, 1)) evaluates to true + # issubclass(object, (1, type)) raises TypeError + for klass in class_seq: + if klass is util.Uninferable: + raise exceptions.AstroidTypeError("arg 2 must be a type or tuple of types") + + for obj_subclass in obj_type.mro(): + if obj_subclass == klass: + return True + return False + + +def object_isinstance(node, class_or_seq, context=None): + """Check if a node 'isinstance' any node in class_or_seq + + :param node: A given node + :param class_or_seq: Union[nodes.NodeNG, Sequence[nodes.NodeNG]] + :rtype: bool + + :raises AstroidTypeError: if the given ``classes_or_seq`` are not types + """ + obj_type = object_type(node, context) + if obj_type is util.Uninferable: + return util.Uninferable + return _object_type_is_subclass(obj_type, class_or_seq, context=context) + + +def object_issubclass(node, class_or_seq, context=None): + """Check if a type is a subclass of any node in class_or_seq + + :param node: A given node + :param class_or_seq: Union[Nodes.NodeNG, Sequence[nodes.NodeNG]] + :rtype: bool + + :raises AstroidTypeError: if the given ``classes_or_seq`` are not types + :raises AstroidError: if the type of the given node cannot be inferred + or its type's mro doesn't work + """ + if not isinstance(node, nodes.ClassDef): + raise TypeError("{node} needs to be a ClassDef node".format(node=node)) + return _object_type_is_subclass(node, class_or_seq, context=context) + + +def safe_infer(node, context=None): + """Return the inferred value for the given node. + + Return None if inference failed or if there is some ambiguity (more than + one node has been inferred). + """ + try: + inferit = node.infer(context=context) + value = next(inferit) + except exceptions.InferenceError: + return None + try: + next(inferit) + return None # None if there is ambiguity on the inferred node + except exceptions.InferenceError: + return None # there is some kind of ambiguity + except StopIteration: + return value + + +def has_known_bases(klass, context=None): + """Return true if all base classes of a class could be inferred.""" + try: + return klass._all_bases_known + except AttributeError: + pass + for base in klass.bases: + result = safe_infer(base, context=context) + # TODO: check for A->B->A->B pattern in class structure too? + if ( + not isinstance(result, scoped_nodes.ClassDef) + or result is klass + or not has_known_bases(result, context=context) + ): + klass._all_bases_known = False + return False + klass._all_bases_known = True + return True + + +def _type_check(type1, type2): + if not all(map(has_known_bases, (type1, type2))): + raise exceptions._NonDeducibleTypeHierarchy + + if not all([type1.newstyle, type2.newstyle]): + return False + try: + return type1 in type2.mro()[:-1] + except exceptions.MroError: + # The MRO is invalid. + raise exceptions._NonDeducibleTypeHierarchy + + +def is_subtype(type1, type2): + """Check if *type1* is a subtype of *type2*.""" + return _type_check(type1=type2, type2=type1) + + +def is_supertype(type1, type2): + """Check if *type2* is a supertype of *type1*.""" + return _type_check(type1, type2) + + +def class_instance_as_index(node): + """Get the value as an index for the given instance. + + If an instance provides an __index__ method, then it can + be used in some scenarios where an integer is expected, + for instance when multiplying or subscripting a list. + """ + context = contextmod.InferenceContext() + context.callcontext = contextmod.CallContext(args=[node]) + + try: + for inferred in node.igetattr("__index__", context=context): + if not isinstance(inferred, bases.BoundMethod): + continue + + for result in inferred.infer_call_result(node, context=context): + if isinstance(result, nodes.Const) and isinstance(result.value, int): + return result + except exceptions.InferenceError: + pass + return None + + +def object_len(node, context=None): + """Infer length of given node object + + :param Union[nodes.ClassDef, nodes.Instance] node: + :param node: Node to infer length of + + :raises AstroidTypeError: If an invalid node is returned + from __len__ method or no __len__ method exists + :raises InferenceError: If the given node cannot be inferred + or if multiple nodes are inferred + :rtype int: Integer length of node + """ + # pylint: disable=import-outside-toplevel; circular import + from astroid.objects import FrozenSet + + inferred_node = safe_infer(node, context=context) + if inferred_node is None or inferred_node is util.Uninferable: + raise exceptions.InferenceError(node=node) + if isinstance(inferred_node, nodes.Const) and isinstance( + inferred_node.value, (bytes, str) + ): + return len(inferred_node.value) + if isinstance(inferred_node, (nodes.List, nodes.Set, nodes.Tuple, FrozenSet)): + return len(inferred_node.elts) + if isinstance(inferred_node, nodes.Dict): + return len(inferred_node.items) + + node_type = object_type(inferred_node, context=context) + if not node_type: + raise exceptions.InferenceError(node=node) + + try: + len_call = next(node_type.igetattr("__len__", context=context)) + except exceptions.AttributeInferenceError: + raise exceptions.AstroidTypeError( + "object of type '{}' has no len()".format(node_type.pytype()) + ) + + result_of_len = next(len_call.infer_call_result(node, context)) + if ( + isinstance(result_of_len, nodes.Const) + and result_of_len.pytype() == "builtins.int" + ): + return result_of_len.value + if isinstance(result_of_len, bases.Instance) and result_of_len.is_subtype_of( + "builtins.int" + ): + # Fake a result as we don't know the arguments of the instance call. + return 0 + raise exceptions.AstroidTypeError( + "'{}' object cannot be interpreted as an integer".format(result_of_len) + ) diff --git a/WebCrawler/venv/Lib/site-packages/astroid/inference.py b/WebCrawler/venv/Lib/site-packages/astroid/inference.py new file mode 100644 index 0000000..bc3e1f9 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/astroid/inference.py @@ -0,0 +1,994 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2006-2011, 2013-2014 LOGILAB S.A. (Paris, FRANCE) +# Copyright (c) 2012 FELD Boris +# Copyright (c) 2013-2014 Google, Inc. +# Copyright (c) 2014-2020 Claudiu Popa +# Copyright (c) 2014 Eevee (Alex Munroe) +# Copyright (c) 2015-2016 Ceridwen +# Copyright (c) 2015 Dmitry Pribysh +# Copyright (c) 2016 Jakub Wilk +# Copyright (c) 2017 Michał Masłowski +# Copyright (c) 2017 Calen Pennington +# Copyright (c) 2017 Łukasz Rogalski +# Copyright (c) 2018-2019 Nick Drozd +# Copyright (c) 2018 Daniel Martin +# Copyright (c) 2018 Ville Skyttä +# Copyright (c) 2018 Bryce Guinta +# Copyright (c) 2018 Ashley Whetter +# Copyright (c) 2018 HoverHell +# Copyright (c) 2020 Leandro T. C. Melo + +# Licensed under the LGPL: https://www.gnu.org/licenses/old-licenses/lgpl-2.1.en.html +# For details: https://github.com/PyCQA/astroid/blob/master/COPYING.LESSER + +"""this module contains a set of functions to handle inference on astroid trees +""" + +import functools +import itertools +import operator + +import wrapt +from astroid import bases +from astroid import context as contextmod +from astroid import exceptions +from astroid import decorators +from astroid import helpers +from astroid import manager +from astroid import nodes +from astroid.interpreter import dunder_lookup +from astroid import protocols +from astroid import util + + +MANAGER = manager.AstroidManager() +# Prevents circular imports +objects = util.lazy_import("objects") + + +# .infer method ############################################################### + + +def infer_end(self, context=None): + """Inference's end for nodes that yield themselves on inference + + These are objects for which inference does not have any semantic, + such as Module or Consts. + """ + yield self + + +nodes.Module._infer = infer_end +nodes.ClassDef._infer = infer_end +nodes.Lambda._infer = infer_end +nodes.Const._infer = infer_end +nodes.Slice._infer = infer_end + + +def _infer_sequence_helper(node, context=None): + """Infer all values based on _BaseContainer.elts""" + values = [] + + for elt in node.elts: + if isinstance(elt, nodes.Starred): + starred = helpers.safe_infer(elt.value, context) + if not starred: + raise exceptions.InferenceError(node=node, context=context) + if not hasattr(starred, "elts"): + raise exceptions.InferenceError(node=node, context=context) + values.extend(_infer_sequence_helper(starred)) + elif isinstance(elt, nodes.NamedExpr): + value = helpers.safe_infer(elt.value, context) + if not value: + raise exceptions.InferenceError(node=node, context=context) + values.append(value) + else: + values.append(elt) + return values + + +@decorators.raise_if_nothing_inferred +def infer_sequence(self, context=None): + has_starred_named_expr = any( + isinstance(e, (nodes.Starred, nodes.NamedExpr)) for e in self.elts + ) + if has_starred_named_expr: + values = _infer_sequence_helper(self, context) + new_seq = type(self)( + lineno=self.lineno, col_offset=self.col_offset, parent=self.parent + ) + new_seq.postinit(values) + + yield new_seq + else: + yield self + + +nodes.List._infer = infer_sequence +nodes.Tuple._infer = infer_sequence +nodes.Set._infer = infer_sequence + + +def infer_map(self, context=None): + if not any(isinstance(k, nodes.DictUnpack) for k, _ in self.items): + yield self + else: + items = _infer_map(self, context) + new_seq = type(self)(self.lineno, self.col_offset, self.parent) + new_seq.postinit(list(items.items())) + yield new_seq + + +def _update_with_replacement(lhs_dict, rhs_dict): + """Delete nodes that equate to duplicate keys + + Since an astroid node doesn't 'equal' another node with the same value, + this function uses the as_string method to make sure duplicate keys + don't get through + + Note that both the key and the value are astroid nodes + + Fixes issue with DictUnpack causing duplicte keys + in inferred Dict items + + :param dict(nodes.NodeNG, nodes.NodeNG) lhs_dict: Dictionary to 'merge' nodes into + :param dict(nodes.NodeNG, nodes.NodeNG) rhs_dict: Dictionary with nodes to pull from + :return dict(nodes.NodeNG, nodes.NodeNG): merged dictionary of nodes + """ + combined_dict = itertools.chain(lhs_dict.items(), rhs_dict.items()) + # Overwrite keys which have the same string values + string_map = {key.as_string(): (key, value) for key, value in combined_dict} + # Return to dictionary + return dict(string_map.values()) + + +def _infer_map(node, context): + """Infer all values based on Dict.items""" + values = {} + for name, value in node.items: + if isinstance(name, nodes.DictUnpack): + double_starred = helpers.safe_infer(value, context) + if not double_starred: + raise exceptions.InferenceError + if not isinstance(double_starred, nodes.Dict): + raise exceptions.InferenceError(node=node, context=context) + unpack_items = _infer_map(double_starred, context) + values = _update_with_replacement(values, unpack_items) + else: + key = helpers.safe_infer(name, context=context) + value = helpers.safe_infer(value, context=context) + if any(not elem for elem in (key, value)): + raise exceptions.InferenceError(node=node, context=context) + values = _update_with_replacement(values, {key: value}) + return values + + +nodes.Dict._infer = infer_map + + +def _higher_function_scope(node): + """ Search for the first function which encloses the given + scope. This can be used for looking up in that function's + scope, in case looking up in a lower scope for a particular + name fails. + + :param node: A scope node. + :returns: + ``None``, if no parent function scope was found, + otherwise an instance of :class:`astroid.scoped_nodes.Function`, + which encloses the given node. + """ + current = node + while current.parent and not isinstance(current.parent, nodes.FunctionDef): + current = current.parent + if current and current.parent: + return current.parent + return None + + +def infer_name(self, context=None): + """infer a Name: use name lookup rules""" + frame, stmts = self.lookup(self.name) + if not stmts: + # Try to see if the name is enclosed in a nested function + # and use the higher (first function) scope for searching. + parent_function = _higher_function_scope(self.scope()) + if parent_function: + _, stmts = parent_function.lookup(self.name) + + if not stmts: + raise exceptions.NameInferenceError( + name=self.name, scope=self.scope(), context=context + ) + context = contextmod.copy_context(context) + context.lookupname = self.name + return bases._infer_stmts(stmts, context, frame) + + +# pylint: disable=no-value-for-parameter +nodes.Name._infer = decorators.raise_if_nothing_inferred( + decorators.path_wrapper(infer_name) +) +nodes.AssignName.infer_lhs = infer_name # won't work with a path wrapper + + +@decorators.raise_if_nothing_inferred +@decorators.path_wrapper +def infer_call(self, context=None): + """infer a Call node by trying to guess what the function returns""" + callcontext = contextmod.copy_context(context) + callcontext.callcontext = contextmod.CallContext( + args=self.args, keywords=self.keywords + ) + callcontext.boundnode = None + if context is not None: + callcontext.extra_context = _populate_context_lookup(self, context.clone()) + + for callee in self.func.infer(context): + if callee is util.Uninferable: + yield callee + continue + try: + if hasattr(callee, "infer_call_result"): + yield from callee.infer_call_result(caller=self, context=callcontext) + except exceptions.InferenceError: + continue + return dict(node=self, context=context) + + +nodes.Call._infer = infer_call + + +@decorators.raise_if_nothing_inferred +@decorators.path_wrapper +def infer_import(self, context=None, asname=True): + """infer an Import node: return the imported module/object""" + name = context.lookupname + if name is None: + raise exceptions.InferenceError(node=self, context=context) + + try: + if asname: + yield self.do_import_module(self.real_name(name)) + else: + yield self.do_import_module(name) + except exceptions.AstroidBuildingError as exc: + raise exceptions.InferenceError(node=self, context=context) from exc + + +nodes.Import._infer = infer_import + + +@decorators.raise_if_nothing_inferred +@decorators.path_wrapper +def infer_import_from(self, context=None, asname=True): + """infer a ImportFrom node: return the imported module/object""" + name = context.lookupname + if name is None: + raise exceptions.InferenceError(node=self, context=context) + if asname: + name = self.real_name(name) + + try: + module = self.do_import_module() + except exceptions.AstroidBuildingError as exc: + raise exceptions.InferenceError(node=self, context=context) from exc + + try: + context = contextmod.copy_context(context) + context.lookupname = name + stmts = module.getattr(name, ignore_locals=module is self.root()) + return bases._infer_stmts(stmts, context) + except exceptions.AttributeInferenceError as error: + raise exceptions.InferenceError( + error.message, target=self, attribute=name, context=context + ) from error + + +nodes.ImportFrom._infer = infer_import_from + + +def infer_attribute(self, context=None): + """infer an Attribute node by using getattr on the associated object""" + for owner in self.expr.infer(context): + if owner is util.Uninferable: + yield owner + continue + + if context and context.boundnode: + # This handles the situation where the attribute is accessed through a subclass + # of a base class and the attribute is defined at the base class's level, + # by taking in consideration a redefinition in the subclass. + if isinstance(owner, bases.Instance) and isinstance( + context.boundnode, bases.Instance + ): + try: + if helpers.is_subtype( + helpers.object_type(context.boundnode), + helpers.object_type(owner), + ): + owner = context.boundnode + except exceptions._NonDeducibleTypeHierarchy: + # Can't determine anything useful. + pass + elif not context: + context = contextmod.InferenceContext() + + try: + context.boundnode = owner + yield from owner.igetattr(self.attrname, context) + except ( + exceptions.AttributeInferenceError, + exceptions.InferenceError, + AttributeError, + ): + pass + finally: + context.boundnode = None + return dict(node=self, context=context) + + +nodes.Attribute._infer = decorators.raise_if_nothing_inferred( + decorators.path_wrapper(infer_attribute) +) +# won't work with a path wrapper +nodes.AssignAttr.infer_lhs = decorators.raise_if_nothing_inferred(infer_attribute) + + +@decorators.raise_if_nothing_inferred +@decorators.path_wrapper +def infer_global(self, context=None): + if context.lookupname is None: + raise exceptions.InferenceError(node=self, context=context) + try: + return bases._infer_stmts(self.root().getattr(context.lookupname), context) + except exceptions.AttributeInferenceError as error: + raise exceptions.InferenceError( + error.message, target=self, attribute=context.lookupname, context=context + ) from error + + +nodes.Global._infer = infer_global + + +_SUBSCRIPT_SENTINEL = object() + + +def infer_subscript(self, context=None): + """Inference for subscripts + + We're understanding if the index is a Const + or a slice, passing the result of inference + to the value's `getitem` method, which should + handle each supported index type accordingly. + """ + + found_one = False + for value in self.value.infer(context): + if value is util.Uninferable: + yield util.Uninferable + return None + for index in self.slice.infer(context): + if index is util.Uninferable: + yield util.Uninferable + return None + + # Try to deduce the index value. + index_value = _SUBSCRIPT_SENTINEL + if value.__class__ == bases.Instance: + index_value = index + elif index.__class__ == bases.Instance: + instance_as_index = helpers.class_instance_as_index(index) + if instance_as_index: + index_value = instance_as_index + else: + index_value = index + + if index_value is _SUBSCRIPT_SENTINEL: + raise exceptions.InferenceError(node=self, context=context) + + try: + assigned = value.getitem(index_value, context) + except ( + exceptions.AstroidTypeError, + exceptions.AstroidIndexError, + exceptions.AttributeInferenceError, + AttributeError, + ) as exc: + raise exceptions.InferenceError(node=self, context=context) from exc + + # Prevent inferring if the inferred subscript + # is the same as the original subscripted object. + if self is assigned or assigned is util.Uninferable: + yield util.Uninferable + return None + yield from assigned.infer(context) + found_one = True + + if found_one: + return dict(node=self, context=context) + return None + + +nodes.Subscript._infer = decorators.raise_if_nothing_inferred( + decorators.path_wrapper(infer_subscript) +) +nodes.Subscript.infer_lhs = decorators.raise_if_nothing_inferred(infer_subscript) + + +@decorators.raise_if_nothing_inferred +@decorators.path_wrapper +def _infer_boolop(self, context=None): + """Infer a boolean operation (and / or / not). + + The function will calculate the boolean operation + for all pairs generated through inference for each component + node. + """ + values = self.values + if self.op == "or": + predicate = operator.truth + else: + predicate = operator.not_ + + try: + values = [value.infer(context=context) for value in values] + except exceptions.InferenceError: + yield util.Uninferable + return None + + for pair in itertools.product(*values): + if any(item is util.Uninferable for item in pair): + # Can't infer the final result, just yield Uninferable. + yield util.Uninferable + continue + + bool_values = [item.bool_value() for item in pair] + if any(item is util.Uninferable for item in bool_values): + # Can't infer the final result, just yield Uninferable. + yield util.Uninferable + continue + + # Since the boolean operations are short circuited operations, + # this code yields the first value for which the predicate is True + # and if no value respected the predicate, then the last value will + # be returned (or Uninferable if there was no last value). + # This is conforming to the semantics of `and` and `or`: + # 1 and 0 -> 1 + # 0 and 1 -> 0 + # 1 or 0 -> 1 + # 0 or 1 -> 1 + value = util.Uninferable + for value, bool_value in zip(pair, bool_values): + if predicate(bool_value): + yield value + break + else: + yield value + + return dict(node=self, context=context) + + +nodes.BoolOp._infer = _infer_boolop + + +# UnaryOp, BinOp and AugAssign inferences + + +def _filter_operation_errors(self, infer_callable, context, error): + for result in infer_callable(self, context): + if isinstance(result, error): + # For the sake of .infer(), we don't care about operation + # errors, which is the job of pylint. So return something + # which shows that we can't infer the result. + yield util.Uninferable + else: + yield result + + +def _infer_unaryop(self, context=None): + """Infer what an UnaryOp should return when evaluated.""" + for operand in self.operand.infer(context): + try: + yield operand.infer_unary_op(self.op) + except TypeError as exc: + # The operand doesn't support this operation. + yield util.BadUnaryOperationMessage(operand, self.op, exc) + except AttributeError as exc: + meth = protocols.UNARY_OP_METHOD[self.op] + if meth is None: + # `not node`. Determine node's boolean + # value and negate its result, unless it is + # Uninferable, which will be returned as is. + bool_value = operand.bool_value() + if bool_value is not util.Uninferable: + yield nodes.const_factory(not bool_value) + else: + yield util.Uninferable + else: + if not isinstance(operand, (bases.Instance, nodes.ClassDef)): + # The operation was used on something which + # doesn't support it. + yield util.BadUnaryOperationMessage(operand, self.op, exc) + continue + + try: + try: + methods = dunder_lookup.lookup(operand, meth) + except exceptions.AttributeInferenceError: + yield util.BadUnaryOperationMessage(operand, self.op, exc) + continue + + meth = methods[0] + inferred = next(meth.infer(context=context)) + if inferred is util.Uninferable or not inferred.callable(): + continue + + context = contextmod.copy_context(context) + context.callcontext = contextmod.CallContext(args=[operand]) + call_results = inferred.infer_call_result(self, context=context) + result = next(call_results, None) + if result is None: + # Failed to infer, return the same type. + yield operand + else: + yield result + except exceptions.AttributeInferenceError as exc: + # The unary operation special method was not found. + yield util.BadUnaryOperationMessage(operand, self.op, exc) + except exceptions.InferenceError: + yield util.Uninferable + + +@decorators.raise_if_nothing_inferred +@decorators.path_wrapper +def infer_unaryop(self, context=None): + """Infer what an UnaryOp should return when evaluated.""" + yield from _filter_operation_errors( + self, _infer_unaryop, context, util.BadUnaryOperationMessage + ) + return dict(node=self, context=context) + + +nodes.UnaryOp._infer_unaryop = _infer_unaryop +nodes.UnaryOp._infer = infer_unaryop + + +def _is_not_implemented(const): + """Check if the given const node is NotImplemented.""" + return isinstance(const, nodes.Const) and const.value is NotImplemented + + +def _invoke_binop_inference(instance, opnode, op, other, context, method_name): + """Invoke binary operation inference on the given instance.""" + methods = dunder_lookup.lookup(instance, method_name) + context = contextmod.bind_context_to_node(context, instance) + method = methods[0] + inferred = next(method.infer(context=context)) + if inferred is util.Uninferable: + raise exceptions.InferenceError + return instance.infer_binary_op(opnode, op, other, context, inferred) + + +def _aug_op(instance, opnode, op, other, context, reverse=False): + """Get an inference callable for an augmented binary operation.""" + method_name = protocols.AUGMENTED_OP_METHOD[op] + return functools.partial( + _invoke_binop_inference, + instance=instance, + op=op, + opnode=opnode, + other=other, + context=context, + method_name=method_name, + ) + + +def _bin_op(instance, opnode, op, other, context, reverse=False): + """Get an inference callable for a normal binary operation. + + If *reverse* is True, then the reflected method will be used instead. + """ + if reverse: + method_name = protocols.REFLECTED_BIN_OP_METHOD[op] + else: + method_name = protocols.BIN_OP_METHOD[op] + return functools.partial( + _invoke_binop_inference, + instance=instance, + op=op, + opnode=opnode, + other=other, + context=context, + method_name=method_name, + ) + + +def _get_binop_contexts(context, left, right): + """Get contexts for binary operations. + + This will return two inference contexts, the first one + for x.__op__(y), the other one for y.__rop__(x), where + only the arguments are inversed. + """ + # The order is important, since the first one should be + # left.__op__(right). + for arg in (right, left): + new_context = context.clone() + new_context.callcontext = contextmod.CallContext(args=[arg]) + new_context.boundnode = None + yield new_context + + +def _same_type(type1, type2): + """Check if type1 is the same as type2.""" + return type1.qname() == type2.qname() + + +def _get_binop_flow( + left, left_type, binary_opnode, right, right_type, context, reverse_context +): + """Get the flow for binary operations. + + The rules are a bit messy: + + * if left and right have the same type, then only one + method will be called, left.__op__(right) + * if left and right are unrelated typewise, then first + left.__op__(right) is tried and if this does not exist + or returns NotImplemented, then right.__rop__(left) is tried. + * if left is a subtype of right, then only left.__op__(right) + is tried. + * if left is a supertype of right, then right.__rop__(left) + is first tried and then left.__op__(right) + """ + op = binary_opnode.op + if _same_type(left_type, right_type): + methods = [_bin_op(left, binary_opnode, op, right, context)] + elif helpers.is_subtype(left_type, right_type): + methods = [_bin_op(left, binary_opnode, op, right, context)] + elif helpers.is_supertype(left_type, right_type): + methods = [ + _bin_op(right, binary_opnode, op, left, reverse_context, reverse=True), + _bin_op(left, binary_opnode, op, right, context), + ] + else: + methods = [ + _bin_op(left, binary_opnode, op, right, context), + _bin_op(right, binary_opnode, op, left, reverse_context, reverse=True), + ] + return methods + + +def _get_aug_flow( + left, left_type, aug_opnode, right, right_type, context, reverse_context +): + """Get the flow for augmented binary operations. + + The rules are a bit messy: + + * if left and right have the same type, then left.__augop__(right) + is first tried and then left.__op__(right). + * if left and right are unrelated typewise, then + left.__augop__(right) is tried, then left.__op__(right) + is tried and then right.__rop__(left) is tried. + * if left is a subtype of right, then left.__augop__(right) + is tried and then left.__op__(right). + * if left is a supertype of right, then left.__augop__(right) + is tried, then right.__rop__(left) and then + left.__op__(right) + """ + bin_op = aug_opnode.op.strip("=") + aug_op = aug_opnode.op + if _same_type(left_type, right_type): + methods = [ + _aug_op(left, aug_opnode, aug_op, right, context), + _bin_op(left, aug_opnode, bin_op, right, context), + ] + elif helpers.is_subtype(left_type, right_type): + methods = [ + _aug_op(left, aug_opnode, aug_op, right, context), + _bin_op(left, aug_opnode, bin_op, right, context), + ] + elif helpers.is_supertype(left_type, right_type): + methods = [ + _aug_op(left, aug_opnode, aug_op, right, context), + _bin_op(right, aug_opnode, bin_op, left, reverse_context, reverse=True), + _bin_op(left, aug_opnode, bin_op, right, context), + ] + else: + methods = [ + _aug_op(left, aug_opnode, aug_op, right, context), + _bin_op(left, aug_opnode, bin_op, right, context), + _bin_op(right, aug_opnode, bin_op, left, reverse_context, reverse=True), + ] + return methods + + +def _infer_binary_operation(left, right, binary_opnode, context, flow_factory): + """Infer a binary operation between a left operand and a right operand + + This is used by both normal binary operations and augmented binary + operations, the only difference is the flow factory used. + """ + + context, reverse_context = _get_binop_contexts(context, left, right) + left_type = helpers.object_type(left) + right_type = helpers.object_type(right) + methods = flow_factory( + left, left_type, binary_opnode, right, right_type, context, reverse_context + ) + for method in methods: + try: + results = list(method()) + except AttributeError: + continue + except exceptions.AttributeInferenceError: + continue + except exceptions.InferenceError: + yield util.Uninferable + return + else: + if any(result is util.Uninferable for result in results): + yield util.Uninferable + return + + if all(map(_is_not_implemented, results)): + continue + not_implemented = sum( + 1 for result in results if _is_not_implemented(result) + ) + if not_implemented and not_implemented != len(results): + # Can't infer yet what this is. + yield util.Uninferable + return + + yield from results + return + # The operation doesn't seem to be supported so let the caller know about it + yield util.BadBinaryOperationMessage(left_type, binary_opnode.op, right_type) + + +def _infer_binop(self, context): + """Binary operation inference logic.""" + left = self.left + right = self.right + + # we use two separate contexts for evaluating lhs and rhs because + # 1. evaluating lhs may leave some undesired entries in context.path + # which may not let us infer right value of rhs + context = context or contextmod.InferenceContext() + lhs_context = contextmod.copy_context(context) + rhs_context = contextmod.copy_context(context) + lhs_iter = left.infer(context=lhs_context) + rhs_iter = right.infer(context=rhs_context) + for lhs, rhs in itertools.product(lhs_iter, rhs_iter): + if any(value is util.Uninferable for value in (rhs, lhs)): + # Don't know how to process this. + yield util.Uninferable + return + + try: + yield from _infer_binary_operation(lhs, rhs, self, context, _get_binop_flow) + except exceptions._NonDeducibleTypeHierarchy: + yield util.Uninferable + + +@decorators.yes_if_nothing_inferred +@decorators.path_wrapper +def infer_binop(self, context=None): + return _filter_operation_errors( + self, _infer_binop, context, util.BadBinaryOperationMessage + ) + + +nodes.BinOp._infer_binop = _infer_binop +nodes.BinOp._infer = infer_binop + + +def _infer_augassign(self, context=None): + """Inference logic for augmented binary operations.""" + if context is None: + context = contextmod.InferenceContext() + + rhs_context = context.clone() + + lhs_iter = self.target.infer_lhs(context=context) + rhs_iter = self.value.infer(context=rhs_context) + for lhs, rhs in itertools.product(lhs_iter, rhs_iter): + if any(value is util.Uninferable for value in (rhs, lhs)): + # Don't know how to process this. + yield util.Uninferable + return + + try: + yield from _infer_binary_operation( + left=lhs, + right=rhs, + binary_opnode=self, + context=context, + flow_factory=_get_aug_flow, + ) + except exceptions._NonDeducibleTypeHierarchy: + yield util.Uninferable + + +@decorators.raise_if_nothing_inferred +@decorators.path_wrapper +def infer_augassign(self, context=None): + return _filter_operation_errors( + self, _infer_augassign, context, util.BadBinaryOperationMessage + ) + + +nodes.AugAssign._infer_augassign = _infer_augassign +nodes.AugAssign._infer = infer_augassign + +# End of binary operation inference. + + +@decorators.raise_if_nothing_inferred +def infer_arguments(self, context=None): + name = context.lookupname + if name is None: + raise exceptions.InferenceError(node=self, context=context) + return protocols._arguments_infer_argname(self, name, context) + + +nodes.Arguments._infer = infer_arguments + + +@decorators.raise_if_nothing_inferred +@decorators.path_wrapper +def infer_assign(self, context=None): + """infer a AssignName/AssignAttr: need to inspect the RHS part of the + assign node + """ + if isinstance(self.parent, nodes.AugAssign): + return self.parent.infer(context) + + stmts = list(self.assigned_stmts(context=context)) + return bases._infer_stmts(stmts, context) + + +nodes.AssignName._infer = infer_assign +nodes.AssignAttr._infer = infer_assign + + +@decorators.raise_if_nothing_inferred +@decorators.path_wrapper +def infer_empty_node(self, context=None): + if not self.has_underlying_object(): + yield util.Uninferable + else: + try: + yield from MANAGER.infer_ast_from_something(self.object, context=context) + except exceptions.AstroidError: + yield util.Uninferable + + +nodes.EmptyNode._infer = infer_empty_node + + +@decorators.raise_if_nothing_inferred +def infer_index(self, context=None): + return self.value.infer(context) + + +nodes.Index._infer = infer_index + +# TODO: move directly into bases.Instance when the dependency hell +# will be solved. +def instance_getitem(self, index, context=None): + # Rewrap index to Const for this case + new_context = contextmod.bind_context_to_node(context, self) + if not context: + context = new_context + + # Create a new callcontext for providing index as an argument. + new_context.callcontext = contextmod.CallContext(args=[index]) + + method = next(self.igetattr("__getitem__", context=context), None) + if not isinstance(method, bases.BoundMethod): + raise exceptions.InferenceError( + "Could not find __getitem__ for {node!r}.", node=self, context=context + ) + + return next(method.infer_call_result(self, new_context)) + + +bases.Instance.getitem = instance_getitem + + +def _populate_context_lookup(call, context): + # Allows context to be saved for later + # for inference inside a function + context_lookup = {} + if context is None: + return context_lookup + for arg in call.args: + if isinstance(arg, nodes.Starred): + context_lookup[arg.value] = context + else: + context_lookup[arg] = context + keywords = call.keywords if call.keywords is not None else [] + for keyword in keywords: + context_lookup[keyword.value] = context + return context_lookup + + +@decorators.raise_if_nothing_inferred +def infer_ifexp(self, context=None): + """Support IfExp inference + + If we can't infer the truthiness of the condition, we default + to inferring both branches. Otherwise, we infer either branch + depending on the condition. + """ + both_branches = False + # We use two separate contexts for evaluating lhs and rhs because + # evaluating lhs may leave some undesired entries in context.path + # which may not let us infer right value of rhs. + + context = context or contextmod.InferenceContext() + lhs_context = contextmod.copy_context(context) + rhs_context = contextmod.copy_context(context) + try: + test = next(self.test.infer(context=context.clone())) + except exceptions.InferenceError: + both_branches = True + else: + if test is not util.Uninferable: + if test.bool_value(): + yield from self.body.infer(context=lhs_context) + else: + yield from self.orelse.infer(context=rhs_context) + else: + both_branches = True + if both_branches: + yield from self.body.infer(context=lhs_context) + yield from self.orelse.infer(context=rhs_context) + + +nodes.IfExp._infer = infer_ifexp + + +# pylint: disable=dangerous-default-value +@wrapt.decorator +def _cached_generator(func, instance, args, kwargs, _cache={}): + node = args[0] + try: + return iter(_cache[func, id(node)]) + except KeyError: + result = func(*args, **kwargs) + # Need to keep an iterator around + original, copy = itertools.tee(result) + _cache[func, id(node)] = list(copy) + return original + + +# When inferring a property, we instantiate a new `objects.Property` object, +# which in turn, because it inherits from `FunctionDef`, sets itself in the locals +# of the wrapping frame. This means that everytime we infer a property, the locals +# are mutated with a new instance of the property. This is why we cache the result +# of the function's inference. +@_cached_generator +def infer_functiondef(self, context=None): + if not self.decorators or not bases._is_property(self): + yield self + return dict(node=self, context=context) + + prop_func = objects.Property( + function=self, + name=self.name, + doc=self.doc, + lineno=self.lineno, + parent=self.parent, + col_offset=self.col_offset, + ) + prop_func.postinit(body=[], args=self.args) + yield prop_func + return dict(node=self, context=context) + + +nodes.FunctionDef._infer = infer_functiondef diff --git a/WebCrawler/venv/Lib/site-packages/astroid/interpreter/__init__.py b/WebCrawler/venv/Lib/site-packages/astroid/interpreter/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/WebCrawler/venv/Lib/site-packages/astroid/interpreter/__pycache__/__init__.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/astroid/interpreter/__pycache__/__init__.cpython-39.pyc new file mode 100644 index 0000000..3f30063 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/astroid/interpreter/__pycache__/__init__.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/astroid/interpreter/__pycache__/dunder_lookup.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/astroid/interpreter/__pycache__/dunder_lookup.cpython-39.pyc new file mode 100644 index 0000000..bc3aa34 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/astroid/interpreter/__pycache__/dunder_lookup.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/astroid/interpreter/__pycache__/objectmodel.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/astroid/interpreter/__pycache__/objectmodel.cpython-39.pyc new file mode 100644 index 0000000..ca38575 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/astroid/interpreter/__pycache__/objectmodel.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/astroid/interpreter/_import/__init__.py b/WebCrawler/venv/Lib/site-packages/astroid/interpreter/_import/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/WebCrawler/venv/Lib/site-packages/astroid/interpreter/_import/__pycache__/__init__.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/astroid/interpreter/_import/__pycache__/__init__.cpython-39.pyc new file mode 100644 index 0000000..97a2a68 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/astroid/interpreter/_import/__pycache__/__init__.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/astroid/interpreter/_import/__pycache__/spec.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/astroid/interpreter/_import/__pycache__/spec.cpython-39.pyc new file mode 100644 index 0000000..cfc816d Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/astroid/interpreter/_import/__pycache__/spec.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/astroid/interpreter/_import/__pycache__/util.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/astroid/interpreter/_import/__pycache__/util.cpython-39.pyc new file mode 100644 index 0000000..9039bd9 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/astroid/interpreter/_import/__pycache__/util.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/astroid/interpreter/_import/spec.py b/WebCrawler/venv/Lib/site-packages/astroid/interpreter/_import/spec.py new file mode 100644 index 0000000..3cf5fea --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/astroid/interpreter/_import/spec.py @@ -0,0 +1,346 @@ +# Copyright (c) 2016-2018 Claudiu Popa +# Copyright (c) 2016 Derek Gustafson +# Copyright (c) 2017 Chris Philip +# Copyright (c) 2017 Hugo +# Copyright (c) 2017 ioanatia +# Copyright (c) 2017 Calen Pennington +# Copyright (c) 2018 Nick Drozd +# Copyright (c) 2019 Hugo van Kemenade +# Copyright (c) 2019 Ashley Whetter + +import abc +import collections +import distutils +import enum +import imp +import os +import sys +import zipimport + +try: + import importlib.machinery + + _HAS_MACHINERY = True +except ImportError: + _HAS_MACHINERY = False + +try: + from functools import lru_cache +except ImportError: + from backports.functools_lru_cache import lru_cache + +from . import util + +ModuleType = enum.Enum( + "ModuleType", + "C_BUILTIN C_EXTENSION PKG_DIRECTORY " + "PY_CODERESOURCE PY_COMPILED PY_FROZEN PY_RESOURCE " + "PY_SOURCE PY_ZIPMODULE PY_NAMESPACE", +) +_ImpTypes = { + imp.C_BUILTIN: ModuleType.C_BUILTIN, + imp.C_EXTENSION: ModuleType.C_EXTENSION, + imp.PKG_DIRECTORY: ModuleType.PKG_DIRECTORY, + imp.PY_COMPILED: ModuleType.PY_COMPILED, + imp.PY_FROZEN: ModuleType.PY_FROZEN, + imp.PY_SOURCE: ModuleType.PY_SOURCE, +} +if hasattr(imp, "PY_RESOURCE"): + _ImpTypes[imp.PY_RESOURCE] = ModuleType.PY_RESOURCE +if hasattr(imp, "PY_CODERESOURCE"): + _ImpTypes[imp.PY_CODERESOURCE] = ModuleType.PY_CODERESOURCE + + +def _imp_type_to_module_type(imp_type): + return _ImpTypes[imp_type] + + +_ModuleSpec = collections.namedtuple( + "_ModuleSpec", "name type location " "origin submodule_search_locations" +) + + +class ModuleSpec(_ModuleSpec): + """Defines a class similar to PEP 420's ModuleSpec + + A module spec defines a name of a module, its type, location + and where submodules can be found, if the module is a package. + """ + + def __new__( + cls, + name, + module_type, + location=None, + origin=None, + submodule_search_locations=None, + ): + return _ModuleSpec.__new__( + cls, + name=name, + type=module_type, + location=location, + origin=origin, + submodule_search_locations=submodule_search_locations, + ) + + +class Finder: + """A finder is a class which knows how to find a particular module.""" + + def __init__(self, path=None): + self._path = path or sys.path + + @abc.abstractmethod + def find_module(self, modname, module_parts, processed, submodule_path): + """Find the given module + + Each finder is responsible for each protocol of finding, as long as + they all return a ModuleSpec. + + :param str modname: The module which needs to be searched. + :param list module_parts: It should be a list of strings, + where each part contributes to the module's + namespace. + :param list processed: What parts from the module parts were processed + so far. + :param list submodule_path: A list of paths where the module + can be looked into. + :returns: A ModuleSpec, describing how and where the module was found, + None, otherwise. + """ + + def contribute_to_path(self, spec, processed): + """Get a list of extra paths where this finder can search.""" + + +class ImpFinder(Finder): + """A finder based on the imp module.""" + + def find_module(self, modname, module_parts, processed, submodule_path): + if submodule_path is not None: + submodule_path = list(submodule_path) + try: + stream, mp_filename, mp_desc = imp.find_module(modname, submodule_path) + except ImportError: + return None + + # Close resources. + if stream: + stream.close() + + return ModuleSpec( + name=modname, + location=mp_filename, + module_type=_imp_type_to_module_type(mp_desc[2]), + ) + + def contribute_to_path(self, spec, processed): + if spec.location is None: + # Builtin. + return None + + if _is_setuptools_namespace(spec.location): + # extend_path is called, search sys.path for module/packages + # of this name see pkgutil.extend_path documentation + path = [ + os.path.join(p, *processed) + for p in sys.path + if os.path.isdir(os.path.join(p, *processed)) + ] + # We already import distutils elsewhere in astroid, + # so if it is the same module, we can use it directly. + elif spec.name == "distutils" and spec.location in distutils.__path__: + # distutils is patched inside virtualenvs to pick up submodules + # from the original Python, not from the virtualenv itself. + path = list(distutils.__path__) + else: + path = [spec.location] + return path + + +class ExplicitNamespacePackageFinder(ImpFinder): + """A finder for the explicit namespace packages, generated through pkg_resources.""" + + def find_module(self, modname, module_parts, processed, submodule_path): + if processed: + modname = ".".join(processed + [modname]) + if util.is_namespace(modname) and modname in sys.modules: + submodule_path = sys.modules[modname].__path__ + return ModuleSpec( + name=modname, + location="", + origin="namespace", + module_type=ModuleType.PY_NAMESPACE, + submodule_search_locations=submodule_path, + ) + return None + + def contribute_to_path(self, spec, processed): + return spec.submodule_search_locations + + +class ZipFinder(Finder): + """Finder that knows how to find a module inside zip files.""" + + def __init__(self, path): + super().__init__(path) + self._zipimporters = _precache_zipimporters(path) + + def find_module(self, modname, module_parts, processed, submodule_path): + try: + file_type, filename, path = _search_zip(module_parts, self._zipimporters) + except ImportError: + return None + + return ModuleSpec( + name=modname, + location=filename, + origin="egg", + module_type=file_type, + submodule_search_locations=path, + ) + + +class PathSpecFinder(Finder): + """Finder based on importlib.machinery.PathFinder.""" + + def find_module(self, modname, module_parts, processed, submodule_path): + spec = importlib.machinery.PathFinder.find_spec(modname, path=submodule_path) + if spec: + # origin can be either a string on older Python versions + # or None in case it is a namespace package: + # https://github.com/python/cpython/pull/5481 + is_namespace_pkg = spec.origin in ("namespace", None) + location = spec.origin if not is_namespace_pkg else None + module_type = ModuleType.PY_NAMESPACE if is_namespace_pkg else None + spec = ModuleSpec( + name=spec.name, + location=location, + origin=spec.origin, + module_type=module_type, + submodule_search_locations=list(spec.submodule_search_locations or []), + ) + return spec + + def contribute_to_path(self, spec, processed): + if spec.type == ModuleType.PY_NAMESPACE: + return spec.submodule_search_locations + return None + + +_SPEC_FINDERS = (ImpFinder, ZipFinder) +if _HAS_MACHINERY: + _SPEC_FINDERS += (PathSpecFinder,) +_SPEC_FINDERS += (ExplicitNamespacePackageFinder,) + + +def _is_setuptools_namespace(location): + try: + with open(os.path.join(location, "__init__.py"), "rb") as stream: + data = stream.read(4096) + except IOError: + pass + else: + extend_path = b"pkgutil" in data and b"extend_path" in data + declare_namespace = ( + b"pkg_resources" in data and b"declare_namespace(__name__)" in data + ) + return extend_path or declare_namespace + + +@lru_cache() +def _cached_set_diff(left, right): + result = set(left) + result.difference_update(right) + return result + + +def _precache_zipimporters(path=None): + pic = sys.path_importer_cache + + # When measured, despite having the same complexity (O(n)), + # converting to tuples and then caching the conversion to sets + # and the set difference is faster than converting to sets + # and then only caching the set difference. + + req_paths = tuple(path or sys.path) + cached_paths = tuple(pic) + new_paths = _cached_set_diff(req_paths, cached_paths) + for entry_path in new_paths: + try: + pic[entry_path] = zipimport.zipimporter(entry_path) + except zipimport.ZipImportError: + continue + return pic + + +def _search_zip(modpath, pic): + for filepath, importer in list(pic.items()): + if importer is not None: + found = importer.find_module(modpath[0]) + if found: + if not importer.find_module(os.path.sep.join(modpath)): + raise ImportError( + "No module named %s in %s/%s" + % (".".join(modpath[1:]), filepath, modpath) + ) + # import code; code.interact(local=locals()) + return ( + ModuleType.PY_ZIPMODULE, + os.path.abspath(filepath) + os.path.sep + os.path.sep.join(modpath), + filepath, + ) + raise ImportError("No module named %s" % ".".join(modpath)) + + +def _find_spec_with_path(search_path, modname, module_parts, processed, submodule_path): + finders = [finder(search_path) for finder in _SPEC_FINDERS] + for finder in finders: + spec = finder.find_module(modname, module_parts, processed, submodule_path) + if spec is None: + continue + return finder, spec + + raise ImportError("No module named %s" % ".".join(module_parts)) + + +def find_spec(modpath, path=None): + """Find a spec for the given module. + + :type modpath: list or tuple + :param modpath: + split module's name (i.e name of a module or package split + on '.'), with leading empty strings for explicit relative import + + :type path: list or None + :param path: + optional list of path where the module or package should be + searched (use sys.path if nothing or None is given) + + :rtype: ModuleSpec + :return: A module spec, which describes how the module was + found and where. + """ + _path = path or sys.path + + # Need a copy for not mutating the argument. + modpath = modpath[:] + + submodule_path = None + module_parts = modpath[:] + processed = [] + + while modpath: + modname = modpath.pop(0) + finder, spec = _find_spec_with_path( + _path, modname, module_parts, processed, submodule_path or path + ) + processed.append(modname) + if modpath: + submodule_path = finder.contribute_to_path(spec, processed) + + if spec.type == ModuleType.PKG_DIRECTORY: + spec = spec._replace(submodule_search_locations=submodule_path) + + return spec diff --git a/WebCrawler/venv/Lib/site-packages/astroid/interpreter/_import/util.py b/WebCrawler/venv/Lib/site-packages/astroid/interpreter/_import/util.py new file mode 100644 index 0000000..a917bd3 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/astroid/interpreter/_import/util.py @@ -0,0 +1,10 @@ +# Copyright (c) 2016, 2018 Claudiu Popa + +try: + import pkg_resources +except ImportError: + pkg_resources = None + + +def is_namespace(modname): + return pkg_resources is not None and modname in pkg_resources._namespace_packages diff --git a/WebCrawler/venv/Lib/site-packages/astroid/interpreter/dunder_lookup.py b/WebCrawler/venv/Lib/site-packages/astroid/interpreter/dunder_lookup.py new file mode 100644 index 0000000..0ae9bc9 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/astroid/interpreter/dunder_lookup.py @@ -0,0 +1,66 @@ +# Copyright (c) 2016-2018 Claudiu Popa +# Licensed under the LGPL: https://www.gnu.org/licenses/old-licenses/lgpl-2.1.en.html +# For details: https://github.com/PyCQA/astroid/blob/master/COPYING.LESSER + +"""Contains logic for retrieving special methods. + +This implementation does not rely on the dot attribute access +logic, found in ``.getattr()``. The difference between these two +is that the dunder methods are looked with the type slots +(you can find more about these here +http://lucumr.pocoo.org/2014/8/16/the-python-i-would-like-to-see/) +As such, the lookup for the special methods is actually simpler than +the dot attribute access. +""" +import itertools + +import astroid +from astroid import exceptions + + +def _lookup_in_mro(node, name): + attrs = node.locals.get(name, []) + + nodes = itertools.chain.from_iterable( + ancestor.locals.get(name, []) for ancestor in node.ancestors(recurs=True) + ) + values = list(itertools.chain(attrs, nodes)) + if not values: + raise exceptions.AttributeInferenceError(attribute=name, target=node) + + return values + + +def lookup(node, name): + """Lookup the given special method name in the given *node* + + If the special method was found, then a list of attributes + will be returned. Otherwise, `astroid.AttributeInferenceError` + is going to be raised. + """ + if isinstance( + node, (astroid.List, astroid.Tuple, astroid.Const, astroid.Dict, astroid.Set) + ): + return _builtin_lookup(node, name) + if isinstance(node, astroid.Instance): + return _lookup_in_mro(node, name) + if isinstance(node, astroid.ClassDef): + return _class_lookup(node, name) + + raise exceptions.AttributeInferenceError(attribute=name, target=node) + + +def _class_lookup(node, name): + metaclass = node.metaclass() + if metaclass is None: + raise exceptions.AttributeInferenceError(attribute=name, target=node) + + return _lookup_in_mro(metaclass, name) + + +def _builtin_lookup(node, name): + values = node.locals.get(name, []) + if not values: + raise exceptions.AttributeInferenceError(attribute=name, target=node) + + return values diff --git a/WebCrawler/venv/Lib/site-packages/astroid/interpreter/objectmodel.py b/WebCrawler/venv/Lib/site-packages/astroid/interpreter/objectmodel.py new file mode 100644 index 0000000..277c825 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/astroid/interpreter/objectmodel.py @@ -0,0 +1,801 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2016-2019 Claudiu Popa +# Copyright (c) 2016 Derek Gustafson +# Copyright (c) 2017-2018 Bryce Guinta +# Copyright (c) 2017 Ceridwen +# Copyright (c) 2017 Calen Pennington +# Copyright (c) 2018 Ville Skyttä +# Copyright (c) 2018 Nick Drozd +# Licensed under the LGPL: https://www.gnu.org/licenses/old-licenses/lgpl-2.1.en.html +# For details: https://github.com/PyCQA/astroid/blob/master/COPYING.LESSER +""" +Data object model, as per https://docs.python.org/3/reference/datamodel.html. + +This module describes, at least partially, a data object model for some +of astroid's nodes. The model contains special attributes that nodes such +as functions, classes, modules etc have, such as __doc__, __class__, +__module__ etc, being used when doing attribute lookups over nodes. + +For instance, inferring `obj.__class__` will first trigger an inference +of the `obj` variable. If it was successfully inferred, then an attribute +`__class__ will be looked for in the inferred object. This is the part +where the data model occurs. The model is attached to those nodes +and the lookup mechanism will try to see if attributes such as +`__class__` are defined by the model or not. If they are defined, +the model will be requested to return the corresponding value of that +attribute. Thus the model can be viewed as a special part of the lookup +mechanism. +""" + +import itertools +import pprint +import os +import types +from functools import lru_cache + +import astroid +from astroid import context as contextmod +from astroid import exceptions +from astroid import node_classes + + +IMPL_PREFIX = "attr_" + + +def _dunder_dict(instance, attributes): + obj = node_classes.Dict(parent=instance) + + # Convert the keys to node strings + keys = [ + node_classes.Const(value=value, parent=obj) for value in list(attributes.keys()) + ] + + # The original attribute has a list of elements for each key, + # but that is not useful for retrieving the special attribute's value. + # In this case, we're picking the last value from each list. + values = [elem[-1] for elem in attributes.values()] + + obj.postinit(list(zip(keys, values))) + return obj + + +class ObjectModel: + def __init__(self): + self._instance = None + + def __repr__(self): + result = [] + cname = type(self).__name__ + string = "%(cname)s(%(fields)s)" + alignment = len(cname) + 1 + for field in sorted(self.attributes()): + width = 80 - len(field) - alignment + lines = pprint.pformat(field, indent=2, width=width).splitlines(True) + + inner = [lines[0]] + for line in lines[1:]: + inner.append(" " * alignment + line) + result.append(field) + + return string % { + "cname": cname, + "fields": (",\n" + " " * alignment).join(result), + } + + def __call__(self, instance): + self._instance = instance + return self + + def __get__(self, instance, cls=None): + # ObjectModel needs to be a descriptor so that just doing + # `special_attributes = SomeObjectModel` should be enough in the body of a node. + # But at the same time, node.special_attributes should return an object + # which can be used for manipulating the special attributes. That's the reason + # we pass the instance through which it got accessed to ObjectModel.__call__, + # returning itself afterwards, so we can still have access to the + # underlying data model and to the instance for which it got accessed. + return self(instance) + + def __contains__(self, name): + return name in self.attributes() + + @lru_cache(maxsize=None) + def attributes(self): + """Get the attributes which are exported by this object model.""" + return [ + obj[len(IMPL_PREFIX) :] for obj in dir(self) if obj.startswith(IMPL_PREFIX) + ] + + def lookup(self, name): + """Look up the given *name* in the current model + + It should return an AST or an interpreter object, + but if the name is not found, then an AttributeInferenceError will be raised. + """ + + if name in self.attributes(): + return getattr(self, IMPL_PREFIX + name) + raise exceptions.AttributeInferenceError(target=self._instance, attribute=name) + + +class ModuleModel(ObjectModel): + def _builtins(self): + builtins_ast_module = astroid.MANAGER.builtins_module + return builtins_ast_module.special_attributes.lookup("__dict__") + + @property + def attr_builtins(self): + return self._builtins() + + @property + def attr___path__(self): + if not self._instance.package: + raise exceptions.AttributeInferenceError( + target=self._instance, attribute="__path__" + ) + + path_objs = [ + node_classes.Const( + value=path + if not path.endswith("__init__.py") + else os.path.dirname(path), + parent=self._instance, + ) + for path in self._instance.path + ] + + container = node_classes.List(parent=self._instance) + container.postinit(path_objs) + + return container + + @property + def attr___name__(self): + return node_classes.Const(value=self._instance.name, parent=self._instance) + + @property + def attr___doc__(self): + return node_classes.Const(value=self._instance.doc, parent=self._instance) + + @property + def attr___file__(self): + return node_classes.Const(value=self._instance.file, parent=self._instance) + + @property + def attr___dict__(self): + return _dunder_dict(self._instance, self._instance.globals) + + @property + def attr___package__(self): + if not self._instance.package: + value = "" + else: + value = self._instance.name + + return node_classes.Const(value=value, parent=self._instance) + + # These are related to the Python 3 implementation of the + # import system, + # https://docs.python.org/3/reference/import.html#import-related-module-attributes + + @property + def attr___spec__(self): + # No handling for now. + return node_classes.Unknown() + + @property + def attr___loader__(self): + # No handling for now. + return node_classes.Unknown() + + @property + def attr___cached__(self): + # No handling for now. + return node_classes.Unknown() + + +class FunctionModel(ObjectModel): + @property + def attr___name__(self): + return node_classes.Const(value=self._instance.name, parent=self._instance) + + @property + def attr___doc__(self): + return node_classes.Const(value=self._instance.doc, parent=self._instance) + + @property + def attr___qualname__(self): + return node_classes.Const(value=self._instance.qname(), parent=self._instance) + + @property + def attr___defaults__(self): + func = self._instance + if not func.args.defaults: + return node_classes.Const(value=None, parent=func) + + defaults_obj = node_classes.Tuple(parent=func) + defaults_obj.postinit(func.args.defaults) + return defaults_obj + + @property + def attr___annotations__(self): + obj = node_classes.Dict(parent=self._instance) + + if not self._instance.returns: + returns = None + else: + returns = self._instance.returns + + args = self._instance.args + pair_annotations = itertools.chain( + zip(args.args or [], args.annotations), + zip(args.kwonlyargs, args.kwonlyargs_annotations), + zip(args.posonlyargs or [], args.posonlyargs_annotations), + ) + + annotations = { + arg.name: annotation for (arg, annotation) in pair_annotations if annotation + } + if args.varargannotation: + annotations[args.vararg] = args.varargannotation + if args.kwargannotation: + annotations[args.kwarg] = args.kwargannotation + if returns: + annotations["return"] = returns + + items = [ + (node_classes.Const(key, parent=obj), value) + for (key, value) in annotations.items() + ] + + obj.postinit(items) + return obj + + @property + def attr___dict__(self): + return node_classes.Dict(parent=self._instance) + + attr___globals__ = attr___dict__ + + @property + def attr___kwdefaults__(self): + def _default_args(args, parent): + for arg in args.kwonlyargs: + try: + default = args.default_value(arg.name) + except exceptions.NoDefault: + continue + + name = node_classes.Const(arg.name, parent=parent) + yield name, default + + args = self._instance.args + obj = node_classes.Dict(parent=self._instance) + defaults = dict(_default_args(args, obj)) + + obj.postinit(list(defaults.items())) + return obj + + @property + def attr___module__(self): + return node_classes.Const(self._instance.root().qname()) + + @property + def attr___get__(self): + # pylint: disable=import-outside-toplevel; circular import + from astroid import bases + + func = self._instance + + class DescriptorBoundMethod(bases.BoundMethod): + """Bound method which knows how to understand calling descriptor binding.""" + + def implicit_parameters(self): + # Different than BoundMethod since the signature + # is different. + return 0 + + def infer_call_result(self, caller, context=None): + if len(caller.args) > 2 or len(caller.args) < 1: + raise exceptions.InferenceError( + "Invalid arguments for descriptor binding", + target=self, + context=context, + ) + + context = contextmod.copy_context(context) + cls = next(caller.args[0].infer(context=context)) + + if cls is astroid.Uninferable: + raise exceptions.InferenceError( + "Invalid class inferred", target=self, context=context + ) + + # For some reason func is a Node that the below + # code is not expecting + if isinstance(func, bases.BoundMethod): + yield func + return + + # Rebuild the original value, but with the parent set as the + # class where it will be bound. + new_func = func.__class__( + name=func.name, + doc=func.doc, + lineno=func.lineno, + col_offset=func.col_offset, + ) + # pylint: disable=no-member + new_func.postinit(func.args, func.body, func.decorators, func.returns) + + # Build a proper bound method that points to our newly built function. + proxy = bases.UnboundMethod(new_func) + yield bases.BoundMethod(proxy=proxy, bound=cls) + + @property + def args(self): + """Overwrite the underlying args to match those of the underlying func + + Usually the underlying *func* is a function/method, as in: + + def test(self): + pass + + This has only the *self* parameter but when we access test.__get__ + we get a new object which has two parameters, *self* and *type*. + """ + nonlocal func + positional_or_keyword_params = func.args.args.copy() + positional_or_keyword_params.append(astroid.AssignName(name="type")) + + positional_only_params = func.args.posonlyargs.copy() + + arguments = astroid.Arguments(parent=func.args.parent) + arguments.postinit( + args=positional_or_keyword_params, + posonlyargs=positional_only_params, + defaults=[], + kwonlyargs=[], + kw_defaults=[], + annotations=[], + ) + return arguments + + return DescriptorBoundMethod(proxy=self._instance, bound=self._instance) + + # These are here just for completion. + @property + def attr___ne__(self): + return node_classes.Unknown() + + attr___subclasshook__ = attr___ne__ + attr___str__ = attr___ne__ + attr___sizeof__ = attr___ne__ + attr___setattr___ = attr___ne__ + attr___repr__ = attr___ne__ + attr___reduce__ = attr___ne__ + attr___reduce_ex__ = attr___ne__ + attr___new__ = attr___ne__ + attr___lt__ = attr___ne__ + attr___eq__ = attr___ne__ + attr___gt__ = attr___ne__ + attr___format__ = attr___ne__ + attr___delattr___ = attr___ne__ + attr___getattribute__ = attr___ne__ + attr___hash__ = attr___ne__ + attr___init__ = attr___ne__ + attr___dir__ = attr___ne__ + attr___call__ = attr___ne__ + attr___class__ = attr___ne__ + attr___closure__ = attr___ne__ + attr___code__ = attr___ne__ + + +class ClassModel(ObjectModel): + @property + def attr___module__(self): + return node_classes.Const(self._instance.root().qname()) + + @property + def attr___name__(self): + return node_classes.Const(self._instance.name) + + @property + def attr___qualname__(self): + return node_classes.Const(self._instance.qname()) + + @property + def attr___doc__(self): + return node_classes.Const(self._instance.doc) + + @property + def attr___mro__(self): + if not self._instance.newstyle: + raise exceptions.AttributeInferenceError( + target=self._instance, attribute="__mro__" + ) + + mro = self._instance.mro() + obj = node_classes.Tuple(parent=self._instance) + obj.postinit(mro) + return obj + + @property + def attr_mro(self): + if not self._instance.newstyle: + raise exceptions.AttributeInferenceError( + target=self._instance, attribute="mro" + ) + + # pylint: disable=import-outside-toplevel; circular import + from astroid import bases + + other_self = self + + # Cls.mro is a method and we need to return one in order to have a proper inference. + # The method we're returning is capable of inferring the underlying MRO though. + class MroBoundMethod(bases.BoundMethod): + def infer_call_result(self, caller, context=None): + yield other_self.attr___mro__ + + implicit_metaclass = self._instance.implicit_metaclass() + mro_method = implicit_metaclass.locals["mro"][0] + return MroBoundMethod(proxy=mro_method, bound=implicit_metaclass) + + @property + def attr___bases__(self): + obj = node_classes.Tuple() + context = contextmod.InferenceContext() + elts = list(self._instance._inferred_bases(context)) + obj.postinit(elts=elts) + return obj + + @property + def attr___class__(self): + # pylint: disable=import-outside-toplevel; circular import + from astroid import helpers + + return helpers.object_type(self._instance) + + @property + def attr___subclasses__(self): + """Get the subclasses of the underlying class + + This looks only in the current module for retrieving the subclasses, + thus it might miss a couple of them. + """ + # pylint: disable=import-outside-toplevel; circular import + from astroid import bases + from astroid import scoped_nodes + + if not self._instance.newstyle: + raise exceptions.AttributeInferenceError( + target=self._instance, attribute="__subclasses__" + ) + + qname = self._instance.qname() + root = self._instance.root() + classes = [ + cls + for cls in root.nodes_of_class(scoped_nodes.ClassDef) + if cls != self._instance and cls.is_subtype_of(qname) + ] + + obj = node_classes.List(parent=self._instance) + obj.postinit(classes) + + class SubclassesBoundMethod(bases.BoundMethod): + def infer_call_result(self, caller, context=None): + yield obj + + implicit_metaclass = self._instance.implicit_metaclass() + subclasses_method = implicit_metaclass.locals["__subclasses__"][0] + return SubclassesBoundMethod(proxy=subclasses_method, bound=implicit_metaclass) + + @property + def attr___dict__(self): + return node_classes.Dict(parent=self._instance) + + +class SuperModel(ObjectModel): + @property + def attr___thisclass__(self): + return self._instance.mro_pointer + + @property + def attr___self_class__(self): + return self._instance._self_class + + @property + def attr___self__(self): + return self._instance.type + + @property + def attr___class__(self): + return self._instance._proxied + + +class UnboundMethodModel(ObjectModel): + @property + def attr___class__(self): + # pylint: disable=import-outside-toplevel; circular import + from astroid import helpers + + return helpers.object_type(self._instance) + + @property + def attr___func__(self): + return self._instance._proxied + + @property + def attr___self__(self): + return node_classes.Const(value=None, parent=self._instance) + + attr_im_func = attr___func__ + attr_im_class = attr___class__ + attr_im_self = attr___self__ + + +class BoundMethodModel(FunctionModel): + @property + def attr___func__(self): + return self._instance._proxied._proxied + + @property + def attr___self__(self): + return self._instance.bound + + +class GeneratorModel(FunctionModel): + def __new__(cls, *args, **kwargs): + # Append the values from the GeneratorType unto this object. + ret = super(GeneratorModel, cls).__new__(cls, *args, **kwargs) + generator = astroid.MANAGER.builtins_module["generator"] + for name, values in generator.locals.items(): + method = values[0] + patched = lambda cls, meth=method: meth + + setattr(type(ret), IMPL_PREFIX + name, property(patched)) + + return ret + + @property + def attr___name__(self): + return node_classes.Const( + value=self._instance.parent.name, parent=self._instance + ) + + @property + def attr___doc__(self): + return node_classes.Const( + value=self._instance.parent.doc, parent=self._instance + ) + + +class AsyncGeneratorModel(GeneratorModel): + def __new__(cls, *args, **kwargs): + # Append the values from the AGeneratorType unto this object. + ret = super().__new__(cls, *args, **kwargs) + astroid_builtins = astroid.MANAGER.builtins_module + generator = astroid_builtins.get("async_generator") + if generator is None: + # Make it backward compatible. + generator = astroid_builtins.get("generator") + + for name, values in generator.locals.items(): + method = values[0] + patched = lambda cls, meth=method: meth + + setattr(type(ret), IMPL_PREFIX + name, property(patched)) + + return ret + + +class InstanceModel(ObjectModel): + @property + def attr___class__(self): + return self._instance._proxied + + @property + def attr___module__(self): + return node_classes.Const(self._instance.root().qname()) + + @property + def attr___doc__(self): + return node_classes.Const(self._instance.doc) + + @property + def attr___dict__(self): + return _dunder_dict(self._instance, self._instance.instance_attrs) + + +# Exception instances + + +class ExceptionInstanceModel(InstanceModel): + @property + def attr_args(self): + message = node_classes.Const("") + args = node_classes.Tuple(parent=self._instance) + args.postinit((message,)) + return args + + @property + def attr___traceback__(self): + builtins_ast_module = astroid.MANAGER.builtins_module + traceback_type = builtins_ast_module[types.TracebackType.__name__] + return traceback_type.instantiate_class() + + +class SyntaxErrorInstanceModel(ExceptionInstanceModel): + @property + def attr_text(self): + return node_classes.Const("") + + +class OSErrorInstanceModel(ExceptionInstanceModel): + @property + def attr_filename(self): + return node_classes.Const("") + + @property + def attr_errno(self): + return node_classes.Const(0) + + @property + def attr_strerror(self): + return node_classes.Const("") + + attr_filename2 = attr_filename + + +class ImportErrorInstanceModel(ExceptionInstanceModel): + @property + def attr_name(self): + return node_classes.Const("") + + @property + def attr_path(self): + return node_classes.Const("") + + +BUILTIN_EXCEPTIONS = { + "builtins.SyntaxError": SyntaxErrorInstanceModel, + "builtins.ImportError": ImportErrorInstanceModel, + # These are all similar to OSError in terms of attributes + "builtins.OSError": OSErrorInstanceModel, + "builtins.BlockingIOError": OSErrorInstanceModel, + "builtins.BrokenPipeError": OSErrorInstanceModel, + "builtins.ChildProcessError": OSErrorInstanceModel, + "builtins.ConnectionAbortedError": OSErrorInstanceModel, + "builtins.ConnectionError": OSErrorInstanceModel, + "builtins.ConnectionRefusedError": OSErrorInstanceModel, + "builtins.ConnectionResetError": OSErrorInstanceModel, + "builtins.FileExistsError": OSErrorInstanceModel, + "builtins.FileNotFoundError": OSErrorInstanceModel, + "builtins.InterruptedError": OSErrorInstanceModel, + "builtins.IsADirectoryError": OSErrorInstanceModel, + "builtins.NotADirectoryError": OSErrorInstanceModel, + "builtins.PermissionError": OSErrorInstanceModel, + "builtins.ProcessLookupError": OSErrorInstanceModel, + "builtins.TimeoutError": OSErrorInstanceModel, +} + + +class DictModel(ObjectModel): + @property + def attr___class__(self): + return self._instance._proxied + + def _generic_dict_attribute(self, obj, name): + """Generate a bound method that can infer the given *obj*.""" + + class DictMethodBoundMethod(astroid.BoundMethod): + def infer_call_result(self, caller, context=None): + yield obj + + meth = next(self._instance._proxied.igetattr(name)) + return DictMethodBoundMethod(proxy=meth, bound=self._instance) + + @property + def attr_items(self): + elems = [] + obj = node_classes.List(parent=self._instance) + for key, value in self._instance.items: + elem = node_classes.Tuple(parent=obj) + elem.postinit((key, value)) + elems.append(elem) + obj.postinit(elts=elems) + + # pylint: disable=import-outside-toplevel; circular import + from astroid import objects + + obj = objects.DictItems(obj) + return self._generic_dict_attribute(obj, "items") + + @property + def attr_keys(self): + keys = [key for (key, _) in self._instance.items] + obj = node_classes.List(parent=self._instance) + obj.postinit(elts=keys) + + # pylint: disable=import-outside-toplevel; circular import + from astroid import objects + + obj = objects.DictKeys(obj) + return self._generic_dict_attribute(obj, "keys") + + @property + def attr_values(self): + + values = [value for (_, value) in self._instance.items] + obj = node_classes.List(parent=self._instance) + obj.postinit(values) + + # pylint: disable=import-outside-toplevel; circular import + from astroid import objects + + obj = objects.DictValues(obj) + return self._generic_dict_attribute(obj, "values") + + +class PropertyModel(ObjectModel): + """Model for a builtin property""" + + # pylint: disable=import-outside-toplevel + def _init_function(self, name): + from astroid.node_classes import Arguments + from astroid.scoped_nodes import FunctionDef + + args = Arguments() + args.postinit( + args=[], + defaults=[], + kwonlyargs=[], + kw_defaults=[], + annotations=[], + posonlyargs=[], + posonlyargs_annotations=[], + kwonlyargs_annotations=[], + ) + + function = FunctionDef(name=name, parent=self._instance) + + function.postinit(args=args, body=[]) + return function + + @property + def attr_fget(self): + from astroid.scoped_nodes import FunctionDef + + func = self._instance + + class PropertyFuncAccessor(FunctionDef): + def infer_call_result(self, caller=None, context=None): + nonlocal func + if caller and len(caller.args) != 1: + raise exceptions.InferenceError( + "fget() needs a single argument", target=self, context=context + ) + + yield from func.function.infer_call_result( + caller=caller, context=context + ) + + property_accessor = PropertyFuncAccessor(name="fget", parent=self._instance) + property_accessor.postinit(args=func.args, body=func.body) + return property_accessor + + @property + def attr_setter(self): + return self._init_function("setter") + + @property + def attr_deleter(self): + return self._init_function("deleter") + + @property + def attr_getter(self): + return self._init_function("getter") + + # pylint: enable=import-outside-toplevel diff --git a/WebCrawler/venv/Lib/site-packages/astroid/manager.py b/WebCrawler/venv/Lib/site-packages/astroid/manager.py new file mode 100644 index 0000000..82208ad --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/astroid/manager.py @@ -0,0 +1,350 @@ +# Copyright (c) 2006-2011, 2013-2014 LOGILAB S.A. (Paris, FRANCE) +# Copyright (c) 2014-2019 Claudiu Popa +# Copyright (c) 2014 BioGeek +# Copyright (c) 2014 Google, Inc. +# Copyright (c) 2014 Eevee (Alex Munroe) +# Copyright (c) 2015-2016 Ceridwen +# Copyright (c) 2016 Derek Gustafson +# Copyright (c) 2017 Iva Miholic +# Copyright (c) 2018 Bryce Guinta +# Copyright (c) 2018 Nick Drozd +# Copyright (c) 2019 Raphael Gaschignard +# Copyright (c) 2020 Anubhav <35621759+anubh-v@users.noreply.github.com> +# Copyright (c) 2020 Ashley Whetter + +# Licensed under the LGPL: https://www.gnu.org/licenses/old-licenses/lgpl-2.1.en.html +# For details: https://github.com/PyCQA/astroid/blob/master/COPYING.LESSER + +"""astroid manager: avoid multiple astroid build of a same module when +possible by providing a class responsible to get astroid representation +from various source and using a cache of built modules) +""" + +import os +import zipimport + +from astroid import exceptions +from astroid.interpreter._import import spec +from astroid import modutils +from astroid import transforms + + +ZIP_IMPORT_EXTS = (".zip", ".egg", ".whl") + + +def safe_repr(obj): + try: + return repr(obj) + except Exception: # pylint: disable=broad-except + return "???" + + +class AstroidManager: + """the astroid manager, responsible to build astroid from files + or modules. + + Use the Borg pattern. + """ + + name = "astroid loader" + brain = {} + + def __init__(self): + self.__dict__ = AstroidManager.brain + if not self.__dict__: + # NOTE: cache entries are added by the [re]builder + self.astroid_cache = {} + self._mod_file_cache = {} + self._failed_import_hooks = [] + self.always_load_extensions = False + self.optimize_ast = False + self.extension_package_whitelist = set() + self._transform = transforms.TransformVisitor() + + # Export these APIs for convenience + self.register_transform = self._transform.register_transform + self.unregister_transform = self._transform.unregister_transform + self.max_inferable_values = 100 + + @property + def builtins_module(self): + return self.astroid_cache["builtins"] + + def visit_transforms(self, node): + """Visit the transforms and apply them to the given *node*.""" + return self._transform.visit(node) + + def ast_from_file(self, filepath, modname=None, fallback=True, source=False): + """given a module name, return the astroid object""" + try: + filepath = modutils.get_source_file(filepath, include_no_ext=True) + source = True + except modutils.NoSourceFile: + pass + if modname is None: + try: + modname = ".".join(modutils.modpath_from_file(filepath)) + except ImportError: + modname = filepath + if ( + modname in self.astroid_cache + and self.astroid_cache[modname].file == filepath + ): + return self.astroid_cache[modname] + if source: + # pylint: disable=import-outside-toplevel; circular import + from astroid.builder import AstroidBuilder + + return AstroidBuilder(self).file_build(filepath, modname) + if fallback and modname: + return self.ast_from_module_name(modname) + raise exceptions.AstroidBuildingError( + "Unable to build an AST for {path}.", path=filepath + ) + + def ast_from_string(self, data, modname="", filepath=None): + """ Given some source code as a string, return its corresponding astroid object""" + # pylint: disable=import-outside-toplevel; circular import + from astroid.builder import AstroidBuilder + + return AstroidBuilder(self).string_build(data, modname, filepath) + + def _build_stub_module(self, modname): + # pylint: disable=import-outside-toplevel; circular import + from astroid.builder import AstroidBuilder + + return AstroidBuilder(self).string_build("", modname) + + def _build_namespace_module(self, modname, path): + # pylint: disable=import-outside-toplevel; circular import + from astroid.builder import build_namespace_package_module + + return build_namespace_package_module(modname, path) + + def _can_load_extension(self, modname): + if self.always_load_extensions: + return True + if modutils.is_standard_module(modname): + return True + parts = modname.split(".") + return any( + ".".join(parts[:x]) in self.extension_package_whitelist + for x in range(1, len(parts) + 1) + ) + + def ast_from_module_name(self, modname, context_file=None): + """given a module name, return the astroid object""" + if modname in self.astroid_cache: + return self.astroid_cache[modname] + if modname == "__main__": + return self._build_stub_module(modname) + if context_file: + old_cwd = os.getcwd() + os.chdir(os.path.dirname(context_file)) + try: + found_spec = self.file_from_module_name(modname, context_file) + if found_spec.type == spec.ModuleType.PY_ZIPMODULE: + module = self.zip_import_data(found_spec.location) + if module is not None: + return module + + elif found_spec.type in ( + spec.ModuleType.C_BUILTIN, + spec.ModuleType.C_EXTENSION, + ): + if ( + found_spec.type == spec.ModuleType.C_EXTENSION + and not self._can_load_extension(modname) + ): + return self._build_stub_module(modname) + try: + module = modutils.load_module_from_name(modname) + except Exception as ex: + raise exceptions.AstroidImportError( + "Loading {modname} failed with:\n{error}", + modname=modname, + path=found_spec.location, + ) from ex + return self.ast_from_module(module, modname) + + elif found_spec.type == spec.ModuleType.PY_COMPILED: + raise exceptions.AstroidImportError( + "Unable to load compiled module {modname}.", + modname=modname, + path=found_spec.location, + ) + + elif found_spec.type == spec.ModuleType.PY_NAMESPACE: + return self._build_namespace_module( + modname, found_spec.submodule_search_locations + ) + elif found_spec.type == spec.ModuleType.PY_FROZEN: + return self._build_stub_module(modname) + + if found_spec.location is None: + raise exceptions.AstroidImportError( + "Can't find a file for module {modname}.", modname=modname + ) + + return self.ast_from_file(found_spec.location, modname, fallback=False) + except exceptions.AstroidBuildingError as e: + for hook in self._failed_import_hooks: + try: + return hook(modname) + except exceptions.AstroidBuildingError: + pass + raise e + finally: + if context_file: + os.chdir(old_cwd) + + def zip_import_data(self, filepath): + if zipimport is None: + return None + + # pylint: disable=import-outside-toplevel; circular import + from astroid.builder import AstroidBuilder + + builder = AstroidBuilder(self) + for ext in ZIP_IMPORT_EXTS: + try: + eggpath, resource = filepath.rsplit(ext + os.path.sep, 1) + except ValueError: + continue + try: + importer = zipimport.zipimporter(eggpath + ext) + zmodname = resource.replace(os.path.sep, ".") + if importer.is_package(resource): + zmodname = zmodname + ".__init__" + module = builder.string_build( + importer.get_source(resource), zmodname, filepath + ) + return module + except Exception: # pylint: disable=broad-except + continue + return None + + def file_from_module_name(self, modname, contextfile): + try: + value = self._mod_file_cache[(modname, contextfile)] + except KeyError: + try: + value = modutils.file_info_from_modpath( + modname.split("."), context_file=contextfile + ) + except ImportError as ex: + value = exceptions.AstroidImportError( + "Failed to import module {modname} with error:\n{error}.", + modname=modname, + error=ex, + ) + self._mod_file_cache[(modname, contextfile)] = value + if isinstance(value, exceptions.AstroidBuildingError): + raise value + return value + + def ast_from_module(self, module, modname=None): + """given an imported module, return the astroid object""" + modname = modname or module.__name__ + if modname in self.astroid_cache: + return self.astroid_cache[modname] + try: + # some builtin modules don't have __file__ attribute + filepath = module.__file__ + if modutils.is_python_source(filepath): + return self.ast_from_file(filepath, modname) + except AttributeError: + pass + + # pylint: disable=import-outside-toplevel; circular import + from astroid.builder import AstroidBuilder + + return AstroidBuilder(self).module_build(module, modname) + + def ast_from_class(self, klass, modname=None): + """get astroid for the given class""" + if modname is None: + try: + modname = klass.__module__ + except AttributeError as exc: + raise exceptions.AstroidBuildingError( + "Unable to get module for class {class_name}.", + cls=klass, + class_repr=safe_repr(klass), + modname=modname, + ) from exc + modastroid = self.ast_from_module_name(modname) + return modastroid.getattr(klass.__name__)[0] # XXX + + def infer_ast_from_something(self, obj, context=None): + """infer astroid for the given class""" + if hasattr(obj, "__class__") and not isinstance(obj, type): + klass = obj.__class__ + else: + klass = obj + try: + modname = klass.__module__ + except AttributeError as exc: + raise exceptions.AstroidBuildingError( + "Unable to get module for {class_repr}.", + cls=klass, + class_repr=safe_repr(klass), + ) from exc + except Exception as exc: + raise exceptions.AstroidImportError( + "Unexpected error while retrieving module for {class_repr}:\n" + "{error}", + cls=klass, + class_repr=safe_repr(klass), + ) from exc + try: + name = klass.__name__ + except AttributeError as exc: + raise exceptions.AstroidBuildingError( + "Unable to get name for {class_repr}:\n", + cls=klass, + class_repr=safe_repr(klass), + ) from exc + except Exception as exc: + raise exceptions.AstroidImportError( + "Unexpected error while retrieving name for {class_repr}:\n" "{error}", + cls=klass, + class_repr=safe_repr(klass), + ) from exc + # take care, on living object __module__ is regularly wrong :( + modastroid = self.ast_from_module_name(modname) + if klass is obj: + for inferred in modastroid.igetattr(name, context): + yield inferred + else: + for inferred in modastroid.igetattr(name, context): + yield inferred.instantiate_class() + + def register_failed_import_hook(self, hook): + """Registers a hook to resolve imports that cannot be found otherwise. + + `hook` must be a function that accepts a single argument `modname` which + contains the name of the module or package that could not be imported. + If `hook` can resolve the import, must return a node of type `astroid.Module`, + otherwise, it must raise `AstroidBuildingError`. + """ + self._failed_import_hooks.append(hook) + + def cache_module(self, module): + """Cache a module if no module with the same name is known yet.""" + self.astroid_cache.setdefault(module.name, module) + + def bootstrap(self): + """Bootstrap the required AST modules needed for the manager to work + + The bootstrap usually involves building the AST for the builtins + module, which is required by the rest of astroid to work correctly. + """ + from astroid import raw_building # pylint: disable=import-outside-toplevel + + raw_building._astroid_bootstrapping() + + def clear_cache(self): + """Clear the underlying cache. Also bootstraps the builtins module.""" + self.astroid_cache.clear() + self.bootstrap() diff --git a/WebCrawler/venv/Lib/site-packages/astroid/mixins.py b/WebCrawler/venv/Lib/site-packages/astroid/mixins.py new file mode 100644 index 0000000..497a840 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/astroid/mixins.py @@ -0,0 +1,160 @@ +# Copyright (c) 2010-2011, 2013-2014 LOGILAB S.A. (Paris, FRANCE) +# Copyright (c) 2014-2016, 2018 Claudiu Popa +# Copyright (c) 2014 Google, Inc. +# Copyright (c) 2014 Eevee (Alex Munroe) +# Copyright (c) 2015-2016 Ceridwen +# Copyright (c) 2015 Florian Bruhin +# Copyright (c) 2016 Jakub Wilk +# Copyright (c) 2018 Nick Drozd + +# Licensed under the LGPL: https://www.gnu.org/licenses/old-licenses/lgpl-2.1.en.html +# For details: https://github.com/PyCQA/astroid/blob/master/COPYING.LESSER + +"""This module contains some mixins for the different nodes. +""" +import itertools + +from astroid import decorators +from astroid import exceptions + + +class BlockRangeMixIn: + """override block range """ + + @decorators.cachedproperty + def blockstart_tolineno(self): + return self.lineno + + def _elsed_block_range(self, lineno, orelse, last=None): + """handle block line numbers range for try/finally, for, if and while + statements + """ + if lineno == self.fromlineno: + return lineno, lineno + if orelse: + if lineno >= orelse[0].fromlineno: + return lineno, orelse[-1].tolineno + return lineno, orelse[0].fromlineno - 1 + return lineno, last or self.tolineno + + +class FilterStmtsMixin: + """Mixin for statement filtering and assignment type""" + + def _get_filtered_stmts(self, _, node, _stmts, mystmt): + """method used in _filter_stmts to get statements and trigger break""" + if self.statement() is mystmt: + # original node's statement is the assignment, only keep + # current node (gen exp, list comp) + return [node], True + return _stmts, False + + def assign_type(self): + return self + + +class AssignTypeMixin: + def assign_type(self): + return self + + def _get_filtered_stmts(self, lookup_node, node, _stmts, mystmt): + """method used in filter_stmts""" + if self is mystmt: + return _stmts, True + if self.statement() is mystmt: + # original node's statement is the assignment, only keep + # current node (gen exp, list comp) + return [node], True + return _stmts, False + + +class ParentAssignTypeMixin(AssignTypeMixin): + def assign_type(self): + return self.parent.assign_type() + + +class ImportFromMixin(FilterStmtsMixin): + """MixIn for From and Import Nodes""" + + def _infer_name(self, frame, name): + return name + + def do_import_module(self, modname=None): + """return the ast for a module whose name is imported by + """ + # handle special case where we are on a package node importing a module + # using the same name as the package, which may end in an infinite loop + # on relative imports + # XXX: no more needed ? + mymodule = self.root() + level = getattr(self, "level", None) # Import as no level + if modname is None: + modname = self.modname + # XXX we should investigate deeper if we really want to check + # importing itself: modname and mymodule.name be relative or absolute + if mymodule.relative_to_absolute_name(modname, level) == mymodule.name: + # FIXME: we used to raise InferenceError here, but why ? + return mymodule + + return mymodule.import_module( + modname, level=level, relative_only=level and level >= 1 + ) + + def real_name(self, asname): + """get name from 'as' name""" + for name, _asname in self.names: + if name == "*": + return asname + if not _asname: + name = name.split(".", 1)[0] + _asname = name + if asname == _asname: + return name + raise exceptions.AttributeInferenceError( + "Could not find original name for {attribute} in {target!r}", + target=self, + attribute=asname, + ) + + +class MultiLineBlockMixin: + """Mixin for nodes with multi-line blocks, e.g. For and FunctionDef. + Note that this does not apply to every node with a `body` field. + For instance, an If node has a multi-line body, but the body of an + IfExpr is not multi-line, and hence cannot contain Return nodes, + Assign nodes, etc. + """ + + @decorators.cachedproperty + def _multi_line_blocks(self): + return tuple(getattr(self, field) for field in self._multi_line_block_fields) + + def _get_return_nodes_skip_functions(self): + for block in self._multi_line_blocks: + for child_node in block: + if child_node.is_function: + continue + yield from child_node._get_return_nodes_skip_functions() + + def _get_yield_nodes_skip_lambdas(self): + for block in self._multi_line_blocks: + for child_node in block: + if child_node.is_lambda: + continue + yield from child_node._get_yield_nodes_skip_lambdas() + + @decorators.cached + def _get_assign_nodes(self): + children_assign_nodes = ( + child_node._get_assign_nodes() + for block in self._multi_line_blocks + for child_node in block + ) + return list(itertools.chain.from_iterable(children_assign_nodes)) + + +class NoChildrenMixin: + """Mixin for nodes with no children, e.g. Pass.""" + + def get_children(self): + yield from () diff --git a/WebCrawler/venv/Lib/site-packages/astroid/modutils.py b/WebCrawler/venv/Lib/site-packages/astroid/modutils.py new file mode 100644 index 0000000..4e6ed86 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/astroid/modutils.py @@ -0,0 +1,690 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2014-2018, 2020 Claudiu Popa +# Copyright (c) 2014 Google, Inc. +# Copyright (c) 2014 Denis Laxalde +# Copyright (c) 2014 LOGILAB S.A. (Paris, FRANCE) +# Copyright (c) 2014 Eevee (Alex Munroe) +# Copyright (c) 2015 Florian Bruhin +# Copyright (c) 2015 Radosław Ganczarek +# Copyright (c) 2016 Derek Gustafson +# Copyright (c) 2016 Jakub Wilk +# Copyright (c) 2016 Ceridwen +# Copyright (c) 2018 Ville Skyttä +# Copyright (c) 2018 Mario Corchero +# Copyright (c) 2018 Mario Corchero +# Copyright (c) 2018 Anthony Sottile +# Copyright (c) 2019 Hugo van Kemenade +# Copyright (c) 2019 markmcclain +# Copyright (c) 2019 BasPH + +# Licensed under the LGPL: https://www.gnu.org/licenses/old-licenses/lgpl-2.1.en.html +# For details: https://github.com/PyCQA/astroid/blob/master/COPYING.LESSER + +"""Python modules manipulation utility functions. + +:type PY_SOURCE_EXTS: tuple(str) +:var PY_SOURCE_EXTS: list of possible python source file extension + +:type STD_LIB_DIRS: set of str +:var STD_LIB_DIRS: directories where standard modules are located + +:type BUILTIN_MODULES: dict +:var BUILTIN_MODULES: dictionary with builtin module names has key +""" +import imp +import os +import platform +import sys +import itertools +from distutils.sysconfig import get_python_lib # pylint: disable=import-error + +# pylint: disable=import-error, no-name-in-module +from distutils.errors import DistutilsPlatformError + +# distutils is replaced by virtualenv with a module that does +# weird path manipulations in order to get to the +# real distutils module. +from typing import Optional, List + +from .interpreter._import import spec +from .interpreter._import import util + +if sys.platform.startswith("win"): + PY_SOURCE_EXTS = ("py", "pyw") + PY_COMPILED_EXTS = ("dll", "pyd") +else: + PY_SOURCE_EXTS = ("py",) + PY_COMPILED_EXTS = ("so",) + + +try: + # The explicit sys.prefix is to work around a patch in virtualenv that + # replaces the 'real' sys.prefix (i.e. the location of the binary) + # with the prefix from which the virtualenv was created. This throws + # off the detection logic for standard library modules, thus the + # workaround. + STD_LIB_DIRS = { + get_python_lib(standard_lib=True, prefix=sys.prefix), + # Take care of installations where exec_prefix != prefix. + get_python_lib(standard_lib=True, prefix=sys.exec_prefix), + get_python_lib(standard_lib=True), + } +# get_python_lib(standard_lib=1) is not available on pypy, set STD_LIB_DIR to +# non-valid path, see https://bugs.pypy.org/issue1164 +except DistutilsPlatformError: + STD_LIB_DIRS = set() + +if os.name == "nt": + STD_LIB_DIRS.add(os.path.join(sys.prefix, "dlls")) + try: + # real_prefix is defined when running inside virtual environments, + # created with the **virtualenv** library. + STD_LIB_DIRS.add(os.path.join(sys.real_prefix, "dlls")) + except AttributeError: + # sys.base_exec_prefix is always defined, but in a virtual environment + # created with the stdlib **venv** module, it points to the original + # installation, if the virtual env is activated. + try: + STD_LIB_DIRS.add(os.path.join(sys.base_exec_prefix, "dlls")) + except AttributeError: + pass + +if platform.python_implementation() == "PyPy": + _root = os.path.join(sys.prefix, "lib_pypy") + STD_LIB_DIRS.add(_root) + try: + # real_prefix is defined when running inside virtualenv. + STD_LIB_DIRS.add(os.path.join(sys.real_prefix, "lib_pypy")) + except AttributeError: + pass + del _root +if os.name == "posix": + # Need the real prefix is we're under a virtualenv, otherwise + # the usual one will do. + try: + prefix = sys.real_prefix + except AttributeError: + prefix = sys.prefix + + def _posix_path(path): + base_python = "python%d.%d" % sys.version_info[:2] + return os.path.join(prefix, path, base_python) + + STD_LIB_DIRS.add(_posix_path("lib")) + if sys.maxsize > 2 ** 32: + # This tries to fix a problem with /usr/lib64 builds, + # where systems are running both 32-bit and 64-bit code + # on the same machine, which reflects into the places where + # standard library could be found. More details can be found + # here http://bugs.python.org/issue1294959. + # An easy reproducing case would be + # https://github.com/PyCQA/pylint/issues/712#issuecomment-163178753 + STD_LIB_DIRS.add(_posix_path("lib64")) + +EXT_LIB_DIRS = {get_python_lib(), get_python_lib(True)} +IS_JYTHON = platform.python_implementation() == "Jython" +BUILTIN_MODULES = dict.fromkeys(sys.builtin_module_names, True) + + +class NoSourceFile(Exception): + """exception raised when we are not able to get a python + source file for a precompiled file + """ + + +def _normalize_path(path): + return os.path.normcase(os.path.abspath(path)) + + +def _canonicalize_path(path): + return os.path.realpath(os.path.expanduser(path)) + + +def _path_from_filename(filename, is_jython=IS_JYTHON): + if not is_jython: + return filename + head, has_pyclass, _ = filename.partition("$py.class") + if has_pyclass: + return head + ".py" + return filename + + +def _handle_blacklist(blacklist, dirnames, filenames): + """remove files/directories in the black list + + dirnames/filenames are usually from os.walk + """ + for norecurs in blacklist: + if norecurs in dirnames: + dirnames.remove(norecurs) + elif norecurs in filenames: + filenames.remove(norecurs) + + +_NORM_PATH_CACHE = {} + + +def _cache_normalize_path(path): + """abspath with caching""" + # _module_file calls abspath on every path in sys.path every time it's + # called; on a larger codebase this easily adds up to half a second just + # assembling path components. This cache alleviates that. + try: + return _NORM_PATH_CACHE[path] + except KeyError: + if not path: # don't cache result for '' + return _normalize_path(path) + result = _NORM_PATH_CACHE[path] = _normalize_path(path) + return result + + +def load_module_from_name(dotted_name, path=None, use_sys=True): + """Load a Python module from its name. + + :type dotted_name: str + :param dotted_name: python name of a module or package + + :type path: list or None + :param path: + optional list of path where the module or package should be + searched (use sys.path if nothing or None is given) + + :type use_sys: bool + :param use_sys: + boolean indicating whether the sys.modules dictionary should be + used or not + + + :raise ImportError: if the module or package is not found + + :rtype: module + :return: the loaded module + """ + return load_module_from_modpath(dotted_name.split("."), path, use_sys) + + +def load_module_from_modpath(parts, path: Optional[List[str]] = None, use_sys=1): + """Load a python module from its split name. + + :type parts: list(str) or tuple(str) + :param parts: + python name of a module or package split on '.' + + :param path: + Optional list of path where the module or package should be + searched (use sys.path if nothing or None is given) + + :type use_sys: bool + :param use_sys: + boolean indicating whether the sys.modules dictionary should be used or not + + :raise ImportError: if the module or package is not found + + :rtype: module + :return: the loaded module + """ + if use_sys: + try: + return sys.modules[".".join(parts)] + except KeyError: + pass + modpath = [] + prevmodule = None + for part in parts: + modpath.append(part) + curname = ".".join(modpath) + module = None + if len(modpath) != len(parts): + # even with use_sys=False, should try to get outer packages from sys.modules + module = sys.modules.get(curname) + elif use_sys: + # because it may have been indirectly loaded through a parent + module = sys.modules.get(curname) + if module is None: + mp_file, mp_filename, mp_desc = imp.find_module(part, path) + module = imp.load_module(curname, mp_file, mp_filename, mp_desc) + # mp_file still needs to be closed. + if mp_file: + mp_file.close() + if prevmodule: + setattr(prevmodule, part, module) + _file = getattr(module, "__file__", "") + prevmodule = module + if not _file and util.is_namespace(curname): + continue + if not _file and len(modpath) != len(parts): + raise ImportError("no module in %s" % ".".join(parts[len(modpath) :])) + path = [os.path.dirname(_file)] + return module + + +def load_module_from_file( + filepath: str, path: Optional[List[str]] = None, use_sys=True +): + """Load a Python module from it's path. + + :type filepath: str + :param filepath: path to the python module or package + + :param Optional[List[str]] path: + Optional list of path where the module or package should be + searched (use sys.path if nothing or None is given) + + :type use_sys: bool + :param use_sys: + boolean indicating whether the sys.modules dictionary should be + used or not + + :raise ImportError: if the module or package is not found + + :rtype: module + :return: the loaded module + """ + modpath = modpath_from_file(filepath) + return load_module_from_modpath(modpath, path, use_sys) + + +def check_modpath_has_init(path, mod_path): + """check there are some __init__.py all along the way""" + modpath = [] + for part in mod_path: + modpath.append(part) + path = os.path.join(path, part) + if not _has_init(path): + old_namespace = util.is_namespace(".".join(modpath)) + if not old_namespace: + return False + return True + + +def _get_relative_base_path(filename, path_to_check): + """Extracts the relative mod path of the file to import from + + Check if a file is within the passed in path and if so, returns the + relative mod path from the one passed in. + + If the filename is no in path_to_check, returns None + + Note this function will look for both abs and realpath of the file, + this allows to find the relative base path even if the file is a + symlink of a file in the passed in path + + Examples: + _get_relative_base_path("/a/b/c/d.py", "/a/b") -> ["c","d"] + _get_relative_base_path("/a/b/c/d.py", "/dev") -> None + """ + importable_path = None + path_to_check = os.path.normcase(path_to_check) + abs_filename = os.path.abspath(filename) + if os.path.normcase(abs_filename).startswith(path_to_check): + importable_path = abs_filename + + real_filename = os.path.realpath(filename) + if os.path.normcase(real_filename).startswith(path_to_check): + importable_path = real_filename + + if importable_path: + base_path = os.path.splitext(importable_path)[0] + relative_base_path = base_path[len(path_to_check) :] + return [pkg for pkg in relative_base_path.split(os.sep) if pkg] + + return None + + +def modpath_from_file_with_callback(filename, path=None, is_package_cb=None): + filename = os.path.expanduser(_path_from_filename(filename)) + for pathname in itertools.chain( + path or [], map(_canonicalize_path, sys.path), sys.path + ): + pathname = _cache_normalize_path(pathname) + if not pathname: + continue + modpath = _get_relative_base_path(filename, pathname) + if not modpath: + continue + if is_package_cb(pathname, modpath[:-1]): + return modpath + + raise ImportError( + "Unable to find module for %s in %s" % (filename, ", \n".join(sys.path)) + ) + + +def modpath_from_file(filename, path=None): + """Get the corresponding split module's name from a filename + + This function will return the name of a module or package split on `.`. + + :type filename: str + :param filename: file's path for which we want the module's name + + :type Optional[List[str]] path: + Optional list of path where the module or package should be + searched (use sys.path if nothing or None is given) + + :raise ImportError: + if the corresponding module's name has not been found + + :rtype: list(str) + :return: the corresponding split module's name + """ + return modpath_from_file_with_callback(filename, path, check_modpath_has_init) + + +def file_from_modpath(modpath, path=None, context_file=None): + return file_info_from_modpath(modpath, path, context_file).location + + +def file_info_from_modpath(modpath, path=None, context_file=None): + """given a mod path (i.e. split module / package name), return the + corresponding file, giving priority to source file over precompiled + file if it exists + + :type modpath: list or tuple + :param modpath: + split module's name (i.e name of a module or package split + on '.') + (this means explicit relative imports that start with dots have + empty strings in this list!) + + :type path: list or None + :param path: + optional list of path where the module or package should be + searched (use sys.path if nothing or None is given) + + :type context_file: str or None + :param context_file: + context file to consider, necessary if the identifier has been + introduced using a relative import unresolvable in the actual + context (i.e. modutils) + + :raise ImportError: if there is no such module in the directory + + :rtype: (str or None, import type) + :return: + the path to the module's file or None if it's an integrated + builtin module such as 'sys' + """ + if context_file is not None: + context = os.path.dirname(context_file) + else: + context = context_file + if modpath[0] == "xml": + # handle _xmlplus + try: + return _spec_from_modpath(["_xmlplus"] + modpath[1:], path, context) + except ImportError: + return _spec_from_modpath(modpath, path, context) + elif modpath == ["os", "path"]: + # FIXME: currently ignoring search_path... + return spec.ModuleSpec( + name="os.path", location=os.path.__file__, module_type=imp.PY_SOURCE + ) + return _spec_from_modpath(modpath, path, context) + + +def get_module_part(dotted_name, context_file=None): + """given a dotted name return the module part of the name : + + >>> get_module_part('astroid.as_string.dump') + 'astroid.as_string' + + :type dotted_name: str + :param dotted_name: full name of the identifier we are interested in + + :type context_file: str or None + :param context_file: + context file to consider, necessary if the identifier has been + introduced using a relative import unresolvable in the actual + context (i.e. modutils) + + + :raise ImportError: if there is no such module in the directory + + :rtype: str or None + :return: + the module part of the name or None if we have not been able at + all to import the given name + + XXX: deprecated, since it doesn't handle package precedence over module + (see #10066) + """ + # os.path trick + if dotted_name.startswith("os.path"): + return "os.path" + parts = dotted_name.split(".") + if context_file is not None: + # first check for builtin module which won't be considered latter + # in that case (path != None) + if parts[0] in BUILTIN_MODULES: + if len(parts) > 2: + raise ImportError(dotted_name) + return parts[0] + # don't use += or insert, we want a new list to be created ! + path = None + starti = 0 + if parts[0] == "": + assert ( + context_file is not None + ), "explicit relative import, but no context_file?" + path = [] # prevent resolving the import non-relatively + starti = 1 + while parts[starti] == "": # for all further dots: change context + starti += 1 + context_file = os.path.dirname(context_file) + for i in range(starti, len(parts)): + try: + file_from_modpath( + parts[starti : i + 1], path=path, context_file=context_file + ) + except ImportError: + if i < max(1, len(parts) - 2): + raise + return ".".join(parts[:i]) + return dotted_name + + +def get_module_files(src_directory, blacklist, list_all=False): + """given a package directory return a list of all available python + module's files in the package and its subpackages + + :type src_directory: str + :param src_directory: + path of the directory corresponding to the package + + :type blacklist: list or tuple + :param blacklist: iterable + list of files or directories to ignore. + + :type list_all: bool + :param list_all: + get files from all paths, including ones without __init__.py + + :rtype: list + :return: + the list of all available python module's files in the package and + its subpackages + """ + files = [] + for directory, dirnames, filenames in os.walk(src_directory): + if directory in blacklist: + continue + _handle_blacklist(blacklist, dirnames, filenames) + # check for __init__.py + if not list_all and "__init__.py" not in filenames: + dirnames[:] = () + continue + for filename in filenames: + if _is_python_file(filename): + src = os.path.join(directory, filename) + files.append(src) + return files + + +def get_source_file(filename, include_no_ext=False): + """given a python module's file name return the matching source file + name (the filename will be returned identically if it's already an + absolute path to a python source file...) + + :type filename: str + :param filename: python module's file name + + + :raise NoSourceFile: if no source file exists on the file system + + :rtype: str + :return: the absolute path of the source file if it exists + """ + filename = os.path.abspath(_path_from_filename(filename)) + base, orig_ext = os.path.splitext(filename) + for ext in PY_SOURCE_EXTS: + source_path = "%s.%s" % (base, ext) + if os.path.exists(source_path): + return source_path + if include_no_ext and not orig_ext and os.path.exists(base): + return base + raise NoSourceFile(filename) + + +def is_python_source(filename): + """ + rtype: bool + return: True if the filename is a python source file + """ + return os.path.splitext(filename)[1][1:] in PY_SOURCE_EXTS + + +def is_standard_module(modname, std_path=None): + """try to guess if a module is a standard python module (by default, + see `std_path` parameter's description) + + :type modname: str + :param modname: name of the module we are interested in + + :type std_path: list(str) or tuple(str) + :param std_path: list of path considered has standard + + + :rtype: bool + :return: + true if the module: + - is located on the path listed in one of the directory in `std_path` + - is a built-in module + """ + modname = modname.split(".")[0] + try: + filename = file_from_modpath([modname]) + except ImportError: + # import failed, i'm probably not so wrong by supposing it's + # not standard... + return False + # modules which are not living in a file are considered standard + # (sys and __builtin__ for instance) + if filename is None: + # we assume there are no namespaces in stdlib + return not util.is_namespace(modname) + filename = _normalize_path(filename) + for path in EXT_LIB_DIRS: + if filename.startswith(_cache_normalize_path(path)): + return False + if std_path is None: + std_path = STD_LIB_DIRS + for path in std_path: + if filename.startswith(_cache_normalize_path(path)): + return True + return False + + +def is_relative(modname, from_file): + """return true if the given module name is relative to the given + file name + + :type modname: str + :param modname: name of the module we are interested in + + :type from_file: str + :param from_file: + path of the module from which modname has been imported + + :rtype: bool + :return: + true if the module has been imported relatively to `from_file` + """ + if not os.path.isdir(from_file): + from_file = os.path.dirname(from_file) + if from_file in sys.path: + return False + try: + stream, _, _ = imp.find_module(modname.split(".")[0], [from_file]) + + # Close the stream to avoid ResourceWarnings. + if stream: + stream.close() + return True + except ImportError: + return False + + +# internal only functions ##################################################### + + +def _spec_from_modpath(modpath, path=None, context=None): + """given a mod path (i.e. split module / package name), return the + corresponding spec + + this function is used internally, see `file_from_modpath`'s + documentation for more information + """ + assert modpath + location = None + if context is not None: + try: + found_spec = spec.find_spec(modpath, [context]) + location = found_spec.location + except ImportError: + found_spec = spec.find_spec(modpath, path) + location = found_spec.location + else: + found_spec = spec.find_spec(modpath, path) + if found_spec.type == spec.ModuleType.PY_COMPILED: + try: + location = get_source_file(found_spec.location) + return found_spec._replace( + location=location, type=spec.ModuleType.PY_SOURCE + ) + except NoSourceFile: + return found_spec._replace(location=location) + elif found_spec.type == spec.ModuleType.C_BUILTIN: + # integrated builtin module + return found_spec._replace(location=None) + elif found_spec.type == spec.ModuleType.PKG_DIRECTORY: + location = _has_init(found_spec.location) + return found_spec._replace(location=location, type=spec.ModuleType.PY_SOURCE) + return found_spec + + +def _is_python_file(filename): + """return true if the given filename should be considered as a python file + + .pyc and .pyo are ignored + """ + return filename.endswith((".py", ".so", ".pyd", ".pyw")) + + +def _has_init(directory): + """if the given directory has a valid __init__ file, return its path, + else return None + """ + mod_or_pack = os.path.join(directory, "__init__") + for ext in PY_SOURCE_EXTS + ("pyc", "pyo"): + if os.path.exists(mod_or_pack + "." + ext): + return mod_or_pack + "." + ext + return None + + +def is_namespace(specobj): + return specobj.type == spec.ModuleType.PY_NAMESPACE + + +def is_directory(specobj): + return specobj.type == spec.ModuleType.PKG_DIRECTORY diff --git a/WebCrawler/venv/Lib/site-packages/astroid/node_classes.py b/WebCrawler/venv/Lib/site-packages/astroid/node_classes.py new file mode 100644 index 0000000..621dc5f --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/astroid/node_classes.py @@ -0,0 +1,4862 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2009-2011, 2013-2014 LOGILAB S.A. (Paris, FRANCE) +# Copyright (c) 2010 Daniel Harding +# Copyright (c) 2012 FELD Boris +# Copyright (c) 2013-2014 Google, Inc. +# Copyright (c) 2014-2020 Claudiu Popa +# Copyright (c) 2014 Eevee (Alex Munroe) +# Copyright (c) 2015-2016 Ceridwen +# Copyright (c) 2015 Florian Bruhin +# Copyright (c) 2016-2017 Derek Gustafson +# Copyright (c) 2016 Jared Garst +# Copyright (c) 2016 Jakub Wilk +# Copyright (c) 2016 Dave Baum +# Copyright (c) 2017-2020 Ashley Whetter +# Copyright (c) 2017, 2019 Łukasz Rogalski +# Copyright (c) 2017 rr- +# Copyright (c) 2018-2019 hippo91 +# Copyright (c) 2018 Nick Drozd +# Copyright (c) 2018 Ville Skyttä +# Copyright (c) 2018 Bryce Guinta +# Copyright (c) 2018 brendanator +# Copyright (c) 2018 HoverHell +# Copyright (c) 2019 kavins14 +# Copyright (c) 2019 kavins14 + +# Licensed under the LGPL: https://www.gnu.org/licenses/old-licenses/lgpl-2.1.en.html +# For details: https://github.com/PyCQA/astroid/blob/master/COPYING.LESSER + +# pylint: disable=too-many-lines; https://github.com/PyCQA/astroid/issues/465 + +"""Module for some node classes. More nodes in scoped_nodes.py +""" + +import abc +import builtins as builtins_mod +import itertools +import pprint +import sys +from functools import lru_cache, singledispatch as _singledispatch + +from astroid import as_string +from astroid import bases +from astroid import context as contextmod +from astroid import decorators +from astroid import exceptions +from astroid import manager +from astroid import mixins +from astroid import util + + +BUILTINS = builtins_mod.__name__ +MANAGER = manager.AstroidManager() +PY38 = sys.version_info[:2] >= (3, 8) + + +def _is_const(value): + return isinstance(value, tuple(CONST_CLS)) + + +@decorators.raise_if_nothing_inferred +def unpack_infer(stmt, context=None): + """recursively generate nodes inferred by the given statement. + If the inferred value is a list or a tuple, recurse on the elements + """ + if isinstance(stmt, (List, Tuple)): + for elt in stmt.elts: + if elt is util.Uninferable: + yield elt + continue + yield from unpack_infer(elt, context) + return dict(node=stmt, context=context) + # if inferred is a final node, return it and stop + inferred = next(stmt.infer(context)) + if inferred is stmt: + yield inferred + return dict(node=stmt, context=context) + # else, infer recursively, except Uninferable object that should be returned as is + for inferred in stmt.infer(context): + if inferred is util.Uninferable: + yield inferred + else: + yield from unpack_infer(inferred, context) + + return dict(node=stmt, context=context) + + +def are_exclusive( + stmt1, stmt2, exceptions=None +): # pylint: disable=redefined-outer-name + """return true if the two given statements are mutually exclusive + + `exceptions` may be a list of exception names. If specified, discard If + branches and check one of the statement is in an exception handler catching + one of the given exceptions. + + algorithm : + 1) index stmt1's parents + 2) climb among stmt2's parents until we find a common parent + 3) if the common parent is a If or TryExcept statement, look if nodes are + in exclusive branches + """ + # index stmt1's parents + stmt1_parents = {} + children = {} + node = stmt1.parent + previous = stmt1 + while node: + stmt1_parents[node] = 1 + children[node] = previous + previous = node + node = node.parent + # climb among stmt2's parents until we find a common parent + node = stmt2.parent + previous = stmt2 + while node: + if node in stmt1_parents: + # if the common parent is a If or TryExcept statement, look if + # nodes are in exclusive branches + if isinstance(node, If) and exceptions is None: + if ( + node.locate_child(previous)[1] + is not node.locate_child(children[node])[1] + ): + return True + elif isinstance(node, TryExcept): + c2attr, c2node = node.locate_child(previous) + c1attr, c1node = node.locate_child(children[node]) + if c1node is not c2node: + first_in_body_caught_by_handlers = ( + c2attr == "handlers" + and c1attr == "body" + and previous.catch(exceptions) + ) + second_in_body_caught_by_handlers = ( + c2attr == "body" + and c1attr == "handlers" + and children[node].catch(exceptions) + ) + first_in_else_other_in_handlers = ( + c2attr == "handlers" and c1attr == "orelse" + ) + second_in_else_other_in_handlers = ( + c2attr == "orelse" and c1attr == "handlers" + ) + if any( + ( + first_in_body_caught_by_handlers, + second_in_body_caught_by_handlers, + first_in_else_other_in_handlers, + second_in_else_other_in_handlers, + ) + ): + return True + elif c2attr == "handlers" and c1attr == "handlers": + return previous is not children[node] + return False + previous = node + node = node.parent + return False + + +# getitem() helpers. + +_SLICE_SENTINEL = object() + + +def _slice_value(index, context=None): + """Get the value of the given slice index.""" + + if isinstance(index, Const): + if isinstance(index.value, (int, type(None))): + return index.value + elif index is None: + return None + else: + # Try to infer what the index actually is. + # Since we can't return all the possible values, + # we'll stop at the first possible value. + try: + inferred = next(index.infer(context=context)) + except exceptions.InferenceError: + pass + else: + if isinstance(inferred, Const): + if isinstance(inferred.value, (int, type(None))): + return inferred.value + + # Use a sentinel, because None can be a valid + # value that this function can return, + # as it is the case for unspecified bounds. + return _SLICE_SENTINEL + + +def _infer_slice(node, context=None): + lower = _slice_value(node.lower, context) + upper = _slice_value(node.upper, context) + step = _slice_value(node.step, context) + if all(elem is not _SLICE_SENTINEL for elem in (lower, upper, step)): + return slice(lower, upper, step) + + raise exceptions.AstroidTypeError( + message="Could not infer slice used in subscript", + node=node, + index=node.parent, + context=context, + ) + + +def _container_getitem(instance, elts, index, context=None): + """Get a slice or an item, using the given *index*, for the given sequence.""" + try: + if isinstance(index, Slice): + index_slice = _infer_slice(index, context=context) + new_cls = instance.__class__() + new_cls.elts = elts[index_slice] + new_cls.parent = instance.parent + return new_cls + if isinstance(index, Const): + return elts[index.value] + except IndexError as exc: + raise exceptions.AstroidIndexError( + message="Index {index!s} out of range", + node=instance, + index=index, + context=context, + ) from exc + except TypeError as exc: + raise exceptions.AstroidTypeError( + message="Type error {error!r}", node=instance, index=index, context=context + ) from exc + + raise exceptions.AstroidTypeError("Could not use %s as subscript index" % index) + + +OP_PRECEDENCE = { + op: precedence + for precedence, ops in enumerate( + [ + ["Lambda"], # lambda x: x + 1 + ["IfExp"], # 1 if True else 2 + ["or"], + ["and"], + ["not"], + ["Compare"], # in, not in, is, is not, <, <=, >, >=, !=, == + ["|"], + ["^"], + ["&"], + ["<<", ">>"], + ["+", "-"], + ["*", "@", "/", "//", "%"], + ["UnaryOp"], # +, -, ~ + ["**"], + ["Await"], + ] + ) + for op in ops +} + + +class NodeNG: + """ A node of the new Abstract Syntax Tree (AST). + + This is the base class for all Astroid node classes. + """ + + is_statement = False + """Whether this node indicates a statement. + + :type: bool + """ + optional_assign = False # True for For (and for Comprehension if py <3.0) + """Whether this node optionally assigns a variable. + + This is for loop assignments because loop won't necessarily perform an + assignment if the loop has no iterations. + This is also the case from comprehensions in Python 2. + + :type: bool + """ + is_function = False # True for FunctionDef nodes + """Whether this node indicates a function. + + :type: bool + """ + is_lambda = False + # Attributes below are set by the builder module or by raw factories + lineno = None + """The line that this node appears on in the source code. + + :type: int or None + """ + col_offset = None + """The column that this node appears on in the source code. + + :type: int or None + """ + parent = None + """The parent node in the syntax tree. + + :type: NodeNG or None + """ + _astroid_fields = () + """Node attributes that contain child nodes. + + This is redefined in most concrete classes. + + :type: tuple(str) + """ + _other_fields = () + """Node attributes that do not contain child nodes. + + :type: tuple(str) + """ + _other_other_fields = () + """Attributes that contain AST-dependent fields. + + :type: tuple(str) + """ + # instance specific inference function infer(node, context) + _explicit_inference = None + + def __init__(self, lineno=None, col_offset=None, parent=None): + """ + :param lineno: The line that this node appears on in the source code. + :type lineno: int or None + + :param col_offset: The column that this node appears on in the + source code. + :type col_offset: int or None + + :param parent: The parent node in the syntax tree. + :type parent: NodeNG or None + """ + self.lineno = lineno + self.col_offset = col_offset + self.parent = parent + + def infer(self, context=None, **kwargs): + """Get a generator of the inferred values. + + This is the main entry point to the inference system. + + .. seealso:: :ref:`inference` + + If the instance has some explicit inference function set, it will be + called instead of the default interface. + + :returns: The inferred values. + :rtype: iterable + """ + if context is not None: + context = context.extra_context.get(self, context) + if self._explicit_inference is not None: + # explicit_inference is not bound, give it self explicitly + try: + # pylint: disable=not-callable + return self._explicit_inference(self, context, **kwargs) + except exceptions.UseInferenceDefault: + pass + + if not context: + return self._infer(context, **kwargs) + + key = (self, context.lookupname, context.callcontext, context.boundnode) + if key in context.inferred: + return iter(context.inferred[key]) + + gen = context.cache_generator(key, self._infer(context, **kwargs)) + return util.limit_inference(gen, MANAGER.max_inferable_values) + + def _repr_name(self): + """Get a name for nice representation. + + This is either :attr:`name`, :attr:`attrname`, or the empty string. + + :returns: The nice name. + :rtype: str + """ + names = {"name", "attrname"} + if all(name not in self._astroid_fields for name in names): + return getattr(self, "name", getattr(self, "attrname", "")) + return "" + + def __str__(self): + rname = self._repr_name() + cname = type(self).__name__ + if rname: + string = "%(cname)s.%(rname)s(%(fields)s)" + alignment = len(cname) + len(rname) + 2 + else: + string = "%(cname)s(%(fields)s)" + alignment = len(cname) + 1 + result = [] + for field in self._other_fields + self._astroid_fields: + value = getattr(self, field) + width = 80 - len(field) - alignment + lines = pprint.pformat(value, indent=2, width=width).splitlines(True) + + inner = [lines[0]] + for line in lines[1:]: + inner.append(" " * alignment + line) + result.append("%s=%s" % (field, "".join(inner))) + + return string % { + "cname": cname, + "rname": rname, + "fields": (",\n" + " " * alignment).join(result), + } + + def __repr__(self): + rname = self._repr_name() + if rname: + string = "<%(cname)s.%(rname)s l.%(lineno)s at 0x%(id)x>" + else: + string = "<%(cname)s l.%(lineno)s at 0x%(id)x>" + return string % { + "cname": type(self).__name__, + "rname": rname, + "lineno": self.fromlineno, + "id": id(self), + } + + def accept(self, visitor): + """Visit this node using the given visitor.""" + func = getattr(visitor, "visit_" + self.__class__.__name__.lower()) + return func(self) + + def get_children(self): + """Get the child nodes below this node. + + :returns: The children. + :rtype: iterable(NodeNG) + """ + for field in self._astroid_fields: + attr = getattr(self, field) + if attr is None: + continue + if isinstance(attr, (list, tuple)): + yield from attr + else: + yield attr + + def last_child(self): + """An optimized version of list(get_children())[-1] + + :returns: The last child, or None if no children exist. + :rtype: NodeNG or None + """ + for field in self._astroid_fields[::-1]: + attr = getattr(self, field) + if not attr: # None or empty listy / tuple + continue + if isinstance(attr, (list, tuple)): + return attr[-1] + + return attr + return None + + def parent_of(self, node): + """Check if this node is the parent of the given node. + + :param node: The node to check if it is the child. + :type node: NodeNG + + :returns: True if this node is the parent of the given node, + False otherwise. + :rtype: bool + """ + parent = node.parent + while parent is not None: + if self is parent: + return True + parent = parent.parent + return False + + def statement(self): + """The first parent node, including self, marked as statement node. + + :returns: The first parent statement. + :rtype: NodeNG + """ + if self.is_statement: + return self + return self.parent.statement() + + def frame(self): + """The first parent frame node. + + A frame node is a :class:`Module`, :class:`FunctionDef`, + or :class:`ClassDef`. + + :returns: The first parent frame node. + :rtype: Module or FunctionDef or ClassDef + """ + return self.parent.frame() + + def scope(self): + """The first parent node defining a new scope. + + :returns: The first parent scope node. + :rtype: Module or FunctionDef or ClassDef or Lambda or GenExpr + """ + if self.parent: + return self.parent.scope() + return None + + def root(self): + """Return the root node of the syntax tree. + + :returns: The root node. + :rtype: Module + """ + if self.parent: + return self.parent.root() + return self + + def child_sequence(self, child): + """Search for the sequence that contains this child. + + :param child: The child node to search sequences for. + :type child: NodeNG + + :returns: The sequence containing the given child node. + :rtype: iterable(NodeNG) + + :raises AstroidError: If no sequence could be found that contains + the given child. + """ + for field in self._astroid_fields: + node_or_sequence = getattr(self, field) + if node_or_sequence is child: + return [node_or_sequence] + # /!\ compiler.ast Nodes have an __iter__ walking over child nodes + if ( + isinstance(node_or_sequence, (tuple, list)) + and child in node_or_sequence + ): + return node_or_sequence + + msg = "Could not find %s in %s's children" + raise exceptions.AstroidError(msg % (repr(child), repr(self))) + + def locate_child(self, child): + """Find the field of this node that contains the given child. + + :param child: The child node to search fields for. + :type child: NodeNG + + :returns: A tuple of the name of the field that contains the child, + and the sequence or node that contains the child node. + :rtype: tuple(str, iterable(NodeNG) or NodeNG) + + :raises AstroidError: If no field could be found that contains + the given child. + """ + for field in self._astroid_fields: + node_or_sequence = getattr(self, field) + # /!\ compiler.ast Nodes have an __iter__ walking over child nodes + if child is node_or_sequence: + return field, child + if ( + isinstance(node_or_sequence, (tuple, list)) + and child in node_or_sequence + ): + return field, node_or_sequence + msg = "Could not find %s in %s's children" + raise exceptions.AstroidError(msg % (repr(child), repr(self))) + + # FIXME : should we merge child_sequence and locate_child ? locate_child + # is only used in are_exclusive, child_sequence one time in pylint. + + def next_sibling(self): + """The next sibling statement node. + + :returns: The next sibling statement node. + :rtype: NodeNG or None + """ + return self.parent.next_sibling() + + def previous_sibling(self): + """The previous sibling statement. + + :returns: The previous sibling statement node. + :rtype: NodeNG or None + """ + return self.parent.previous_sibling() + + # these are lazy because they're relatively expensive to compute for every + # single node, and they rarely get looked at + + @decorators.cachedproperty + def fromlineno(self): + """The first line that this node appears on in the source code. + + :type: int or None + """ + if self.lineno is None: + return self._fixed_source_line() + + return self.lineno + + @decorators.cachedproperty + def tolineno(self): + """The last line that this node appears on in the source code. + + :type: int or None + """ + if not self._astroid_fields: + # can't have children + lastchild = None + else: + lastchild = self.last_child() + if lastchild is None: + return self.fromlineno + + return lastchild.tolineno + + def _fixed_source_line(self): + """Attempt to find the line that this node appears on. + + We need this method since not all nodes have :attr:`lineno` set. + + :returns: The line number of this node, + or None if this could not be determined. + :rtype: int or None + """ + line = self.lineno + _node = self + try: + while line is None: + _node = next(_node.get_children()) + line = _node.lineno + except StopIteration: + _node = self.parent + while _node and line is None: + line = _node.lineno + _node = _node.parent + return line + + def block_range(self, lineno): + """Get a range from the given line number to where this node ends. + + :param lineno: The line number to start the range at. + :type lineno: int + + :returns: The range of line numbers that this node belongs to, + starting at the given line number. + :rtype: tuple(int, int or None) + """ + return lineno, self.tolineno + + def set_local(self, name, stmt): + """Define that the given name is declared in the given statement node. + + This definition is stored on the parent scope node. + + .. seealso:: :meth:`scope` + + :param name: The name that is being defined. + :type name: str + + :param stmt: The statement that defines the given name. + :type stmt: NodeNG + """ + self.parent.set_local(name, stmt) + + def nodes_of_class(self, klass, skip_klass=None): + """Get the nodes (including this one or below) of the given types. + + :param klass: The types of node to search for. + :type klass: builtins.type or tuple(builtins.type) + + :param skip_klass: The types of node to ignore. This is useful to ignore + subclasses of :attr:`klass`. + :type skip_klass: builtins.type or tuple(builtins.type) + + :returns: The node of the given types. + :rtype: iterable(NodeNG) + """ + if isinstance(self, klass): + yield self + + if skip_klass is None: + for child_node in self.get_children(): + yield from child_node.nodes_of_class(klass, skip_klass) + + return + + for child_node in self.get_children(): + if isinstance(child_node, skip_klass): + continue + yield from child_node.nodes_of_class(klass, skip_klass) + + @decorators.cached + def _get_assign_nodes(self): + return [] + + def _get_name_nodes(self): + for child_node in self.get_children(): + yield from child_node._get_name_nodes() + + def _get_return_nodes_skip_functions(self): + yield from () + + def _get_yield_nodes_skip_lambdas(self): + yield from () + + def _infer_name(self, frame, name): + # overridden for ImportFrom, Import, Global, TryExcept and Arguments + pass + + def _infer(self, context=None): + """we don't know how to resolve a statement by default""" + # this method is overridden by most concrete classes + raise exceptions.InferenceError( + "No inference function for {node!r}.", node=self, context=context + ) + + def inferred(self): + """Get a list of the inferred values. + + .. seealso:: :ref:`inference` + + :returns: The inferred values. + :rtype: list + """ + return list(self.infer()) + + def instantiate_class(self): + """Instantiate an instance of the defined class. + + .. note:: + + On anything other than a :class:`ClassDef` this will return self. + + :returns: An instance of the defined class. + :rtype: object + """ + return self + + def has_base(self, node): + """Check if this node inherits from the given type. + + :param node: The node defining the base to look for. + Usually this is a :class:`Name` node. + :type node: NodeNG + """ + return False + + def callable(self): + """Whether this node defines something that is callable. + + :returns: True if this defines something that is callable, + False otherwise. + :rtype: bool + """ + return False + + def eq(self, value): + return False + + def as_string(self): + """Get the source code that this node represents. + + :returns: The source code. + :rtype: str + """ + return as_string.to_code(self) + + def repr_tree( + self, + ids=False, + include_linenos=False, + ast_state=False, + indent=" ", + max_depth=0, + max_width=80, + ): + """Get a string representation of the AST from this node. + + :param ids: If true, includes the ids with the node type names. + :type ids: bool + + :param include_linenos: If true, includes the line numbers and + column offsets. + :type include_linenos: bool + + :param ast_state: If true, includes information derived from + the whole AST like local and global variables. + :type ast_state: bool + + :param indent: A string to use to indent the output string. + :type indent: str + + :param max_depth: If set to a positive integer, won't return + nodes deeper than max_depth in the string. + :type max_depth: int + + :param max_width: Attempt to format the output string to stay + within this number of characters, but can exceed it under some + circumstances. Only positive integer values are valid, the default is 80. + :type max_width: int + + :returns: The string representation of the AST. + :rtype: str + """ + # pylint: disable=too-many-statements + @_singledispatch + def _repr_tree(node, result, done, cur_indent="", depth=1): + """Outputs a representation of a non-tuple/list, non-node that's + contained within an AST, including strings. + """ + lines = pprint.pformat( + node, width=max(max_width - len(cur_indent), 1) + ).splitlines(True) + result.append(lines[0]) + result.extend([cur_indent + line for line in lines[1:]]) + return len(lines) != 1 + + # pylint: disable=unused-variable; doesn't understand singledispatch + @_repr_tree.register(tuple) + @_repr_tree.register(list) + def _repr_seq(node, result, done, cur_indent="", depth=1): + """Outputs a representation of a sequence that's contained within an AST.""" + cur_indent += indent + result.append("[") + if not node: + broken = False + elif len(node) == 1: + broken = _repr_tree(node[0], result, done, cur_indent, depth) + elif len(node) == 2: + broken = _repr_tree(node[0], result, done, cur_indent, depth) + if not broken: + result.append(", ") + else: + result.append(",\n") + result.append(cur_indent) + broken = _repr_tree(node[1], result, done, cur_indent, depth) or broken + else: + result.append("\n") + result.append(cur_indent) + for child in node[:-1]: + _repr_tree(child, result, done, cur_indent, depth) + result.append(",\n") + result.append(cur_indent) + _repr_tree(node[-1], result, done, cur_indent, depth) + broken = True + result.append("]") + return broken + + # pylint: disable=unused-variable; doesn't understand singledispatch + @_repr_tree.register(NodeNG) + def _repr_node(node, result, done, cur_indent="", depth=1): + """Outputs a strings representation of an astroid node.""" + if node in done: + result.append( + indent + + " max_depth: + result.append("...") + return False + depth += 1 + cur_indent += indent + if ids: + result.append("%s<0x%x>(\n" % (type(node).__name__, id(node))) + else: + result.append("%s(" % type(node).__name__) + fields = [] + if include_linenos: + fields.extend(("lineno", "col_offset")) + fields.extend(node._other_fields) + fields.extend(node._astroid_fields) + if ast_state: + fields.extend(node._other_other_fields) + if not fields: + broken = False + elif len(fields) == 1: + result.append("%s=" % fields[0]) + broken = _repr_tree( + getattr(node, fields[0]), result, done, cur_indent, depth + ) + else: + result.append("\n") + result.append(cur_indent) + for field in fields[:-1]: + result.append("%s=" % field) + _repr_tree(getattr(node, field), result, done, cur_indent, depth) + result.append(",\n") + result.append(cur_indent) + result.append("%s=" % fields[-1]) + _repr_tree(getattr(node, fields[-1]), result, done, cur_indent, depth) + broken = True + result.append(")") + return broken + + result = [] + _repr_tree(self, result, set()) + return "".join(result) + + def bool_value(self, context=None): + """Determine the boolean value of this node. + + The boolean value of a node can have three + possible values: + + * False: For instance, empty data structures, + False, empty strings, instances which return + explicitly False from the __nonzero__ / __bool__ + method. + * True: Most of constructs are True by default: + classes, functions, modules etc + * Uninferable: The inference engine is uncertain of the + node's value. + + :returns: The boolean value of this node. + :rtype: bool or Uninferable + """ + return util.Uninferable + + def op_precedence(self): + # Look up by class name or default to highest precedence + return OP_PRECEDENCE.get(self.__class__.__name__, len(OP_PRECEDENCE)) + + def op_left_associative(self): + # Everything is left associative except `**` and IfExp + return True + + +class Statement(NodeNG): + """Statement node adding a few attributes""" + + is_statement = True + """Whether this node indicates a statement. + + :type: bool + """ + + def next_sibling(self): + """The next sibling statement node. + + :returns: The next sibling statement node. + :rtype: NodeNG or None + """ + stmts = self.parent.child_sequence(self) + index = stmts.index(self) + try: + return stmts[index + 1] + except IndexError: + pass + + def previous_sibling(self): + """The previous sibling statement. + + :returns: The previous sibling statement node. + :rtype: NodeNG or None + """ + stmts = self.parent.child_sequence(self) + index = stmts.index(self) + if index >= 1: + return stmts[index - 1] + return None + + +class _BaseContainer( + mixins.ParentAssignTypeMixin, NodeNG, bases.Instance, metaclass=abc.ABCMeta +): + """Base class for Set, FrozenSet, Tuple and List.""" + + _astroid_fields = ("elts",) + + def __init__(self, lineno=None, col_offset=None, parent=None): + """ + :param lineno: The line that this node appears on in the source code. + :type lineno: int or None + + :param col_offset: The column that this node appears on in the + source code. + :type col_offset: int or None + + :param parent: The parent node in the syntax tree. + :type parent: NodeNG or None + """ + self.elts = [] + """The elements in the node. + + :type: list(NodeNG) + """ + + super().__init__(lineno, col_offset, parent) + + def postinit(self, elts): + """Do some setup after initialisation. + + :param elts: The list of elements the that node contains. + :type elts: list(NodeNG) + """ + self.elts = elts + + @classmethod + def from_elements(cls, elts=None): + """Create a node of this type from the given list of elements. + + :param elts: The list of elements that the node should contain. + :type elts: list(NodeNG) + + :returns: A new node containing the given elements. + :rtype: NodeNG + """ + node = cls() + if elts is None: + node.elts = [] + else: + node.elts = [const_factory(e) if _is_const(e) else e for e in elts] + return node + + def itered(self): + """An iterator over the elements this node contains. + + :returns: The contents of this node. + :rtype: iterable(NodeNG) + """ + return self.elts + + def bool_value(self, context=None): + """Determine the boolean value of this node. + + :returns: The boolean value of this node. + :rtype: bool or Uninferable + """ + return bool(self.elts) + + @abc.abstractmethod + def pytype(self): + """Get the name of the type that this node represents. + + :returns: The name of the type. + :rtype: str + """ + + def get_children(self): + yield from self.elts + + +class LookupMixIn: + """Mixin to look up a name in the right scope.""" + + @lru_cache(maxsize=None) + def lookup(self, name): + """Lookup where the given variable is assigned. + + The lookup starts from self's scope. If self is not a frame itself + and the name is found in the inner frame locals, statements will be + filtered to remove ignorable statements according to self's location. + + :param name: The name of the variable to find assignments for. + :type name: str + + :returns: The scope node and the list of assignments associated to the + given name according to the scope where it has been found (locals, + globals or builtin). + :rtype: tuple(str, list(NodeNG)) + """ + return self.scope().scope_lookup(self, name) + + def ilookup(self, name): + """Lookup the inferred values of the given variable. + + :param name: The variable name to find values for. + :type name: str + + :returns: The inferred values of the statements returned from + :meth:`lookup`. + :rtype: iterable + """ + frame, stmts = self.lookup(name) + context = contextmod.InferenceContext() + return bases._infer_stmts(stmts, context, frame) + + def _get_filtered_node_statements(self, nodes): + statements = [(node, node.statement()) for node in nodes] + # Next we check if we have ExceptHandlers that are parent + # of the underlying variable, in which case the last one survives + if len(statements) > 1 and all( + isinstance(stmt, ExceptHandler) for _, stmt in statements + ): + statements = [ + (node, stmt) for node, stmt in statements if stmt.parent_of(self) + ] + return statements + + def _filter_stmts(self, stmts, frame, offset): + """Filter the given list of statements to remove ignorable statements. + + If self is not a frame itself and the name is found in the inner + frame locals, statements will be filtered to remove ignorable + statements according to self's location. + + :param stmts: The statements to filter. + :type stmts: list(NodeNG) + + :param frame: The frame that all of the given statements belong to. + :type frame: NodeNG + + :param offset: The line offset to filter statements up to. + :type offset: int + + :returns: The filtered statements. + :rtype: list(NodeNG) + """ + # if offset == -1, my actual frame is not the inner frame but its parent + # + # class A(B): pass + # + # we need this to resolve B correctly + if offset == -1: + myframe = self.frame().parent.frame() + else: + myframe = self.frame() + # If the frame of this node is the same as the statement + # of this node, then the node is part of a class or + # a function definition and the frame of this node should be the + # the upper frame, not the frame of the definition. + # For more information why this is important, + # see Pylint issue #295. + # For example, for 'b', the statement is the same + # as the frame / scope: + # + # def test(b=1): + # ... + + if self.statement() is myframe and myframe.parent: + myframe = myframe.parent.frame() + mystmt = self.statement() + # line filtering if we are in the same frame + # + # take care node may be missing lineno information (this is the case for + # nodes inserted for living objects) + if myframe is frame and mystmt.fromlineno is not None: + assert mystmt.fromlineno is not None, mystmt + mylineno = mystmt.fromlineno + offset + else: + # disabling lineno filtering + mylineno = 0 + + _stmts = [] + _stmt_parents = [] + statements = self._get_filtered_node_statements(stmts) + + for node, stmt in statements: + # line filtering is on and we have reached our location, break + if stmt.fromlineno > mylineno > 0: + break + # Ignore decorators with the same name as the + # decorated function + # Fixes issue #375 + if mystmt is stmt and is_from_decorator(self): + continue + assert hasattr(node, "assign_type"), ( + node, + node.scope(), + node.scope().locals, + ) + assign_type = node.assign_type() + if node.has_base(self): + break + + _stmts, done = assign_type._get_filtered_stmts(self, node, _stmts, mystmt) + if done: + break + + optional_assign = assign_type.optional_assign + if optional_assign and assign_type.parent_of(self): + # we are inside a loop, loop var assignment is hiding previous + # assignment + _stmts = [node] + _stmt_parents = [stmt.parent] + continue + + if isinstance(assign_type, NamedExpr): + _stmts = [node] + continue + + # XXX comment various branches below!!! + try: + pindex = _stmt_parents.index(stmt.parent) + except ValueError: + pass + else: + # we got a parent index, this means the currently visited node + # is at the same block level as a previously visited node + if _stmts[pindex].assign_type().parent_of(assign_type): + # both statements are not at the same block level + continue + # if currently visited node is following previously considered + # assignment and both are not exclusive, we can drop the + # previous one. For instance in the following code :: + # + # if a: + # x = 1 + # else: + # x = 2 + # print x + # + # we can't remove neither x = 1 nor x = 2 when looking for 'x' + # of 'print x'; while in the following :: + # + # x = 1 + # x = 2 + # print x + # + # we can remove x = 1 when we see x = 2 + # + # moreover, on loop assignment types, assignment won't + # necessarily be done if the loop has no iteration, so we don't + # want to clear previous assignments if any (hence the test on + # optional_assign) + if not (optional_assign or are_exclusive(_stmts[pindex], node)): + if ( + # In case of partial function node, if the statement is different + # from the origin function then it can be deleted otherwise it should + # remain to be able to correctly infer the call to origin function. + not node.is_function + or node.qname() != "PartialFunction" + or node.name != _stmts[pindex].name + ): + del _stmt_parents[pindex] + del _stmts[pindex] + if isinstance(node, AssignName): + if not optional_assign and stmt.parent is mystmt.parent: + _stmts = [] + _stmt_parents = [] + elif isinstance(node, DelName): + _stmts = [] + _stmt_parents = [] + continue + if not are_exclusive(self, node): + _stmts.append(node) + _stmt_parents.append(stmt.parent) + return _stmts + + +# Name classes + + +class AssignName( + mixins.NoChildrenMixin, LookupMixIn, mixins.ParentAssignTypeMixin, NodeNG +): + """Variation of :class:`ast.Assign` representing assignment to a name. + + An :class:`AssignName` is the name of something that is assigned to. + This includes variables defined in a function signature or in a loop. + + >>> node = astroid.extract_node('variable = range(10)') + >>> node + + >>> list(node.get_children()) + [, ] + >>> list(node.get_children())[0].as_string() + 'variable' + """ + + _other_fields = ("name",) + + def __init__(self, name=None, lineno=None, col_offset=None, parent=None): + """ + :param name: The name that is assigned to. + :type name: str or None + + :param lineno: The line that this node appears on in the source code. + :type lineno: int or None + + :param col_offset: The column that this node appears on in the + source code. + :type col_offset: int or None + + :param parent: The parent node in the syntax tree. + :type parent: NodeNG or None + """ + self.name = name + """The name that is assigned to. + + :type: str or None + """ + + super().__init__(lineno, col_offset, parent) + + +class DelName( + mixins.NoChildrenMixin, LookupMixIn, mixins.ParentAssignTypeMixin, NodeNG +): + """Variation of :class:`ast.Delete` representing deletion of a name. + + A :class:`DelName` is the name of something that is deleted. + + >>> node = astroid.extract_node("del variable #@") + >>> list(node.get_children()) + [] + >>> list(node.get_children())[0].as_string() + 'variable' + """ + + _other_fields = ("name",) + + def __init__(self, name=None, lineno=None, col_offset=None, parent=None): + """ + :param name: The name that is being deleted. + :type name: str or None + + :param lineno: The line that this node appears on in the source code. + :type lineno: int or None + + :param col_offset: The column that this node appears on in the + source code. + :type col_offset: int or None + + :param parent: The parent node in the syntax tree. + :type parent: NodeNG or None + """ + self.name = name + """The name that is being deleted. + + :type: str or None + """ + + super().__init__(lineno, col_offset, parent) + + +class Name(mixins.NoChildrenMixin, LookupMixIn, NodeNG): + """Class representing an :class:`ast.Name` node. + + A :class:`Name` node is something that is named, but not covered by + :class:`AssignName` or :class:`DelName`. + + >>> node = astroid.extract_node('range(10)') + >>> node + + >>> list(node.get_children()) + [, ] + >>> list(node.get_children())[0].as_string() + 'range' + """ + + _other_fields = ("name",) + + def __init__(self, name=None, lineno=None, col_offset=None, parent=None): + """ + :param name: The name that this node refers to. + :type name: str or None + + :param lineno: The line that this node appears on in the source code. + :type lineno: int or None + + :param col_offset: The column that this node appears on in the + source code. + :type col_offset: int or None + + :param parent: The parent node in the syntax tree. + :type parent: NodeNG or None + """ + self.name = name + """The name that this node refers to. + + :type: str or None + """ + + super().__init__(lineno, col_offset, parent) + + def _get_name_nodes(self): + yield self + + for child_node in self.get_children(): + yield from child_node._get_name_nodes() + + +class Arguments(mixins.AssignTypeMixin, NodeNG): + """Class representing an :class:`ast.arguments` node. + + An :class:`Arguments` node represents that arguments in a + function definition. + + >>> node = astroid.extract_node('def foo(bar): pass') + >>> node + + >>> node.args + + """ + + # Python 3.4+ uses a different approach regarding annotations, + # each argument is a new class, _ast.arg, which exposes an + # 'annotation' attribute. In astroid though, arguments are exposed + # as is in the Arguments node and the only way to expose annotations + # is by using something similar with Python 3.3: + # - we expose 'varargannotation' and 'kwargannotation' of annotations + # of varargs and kwargs. + # - we expose 'annotation', a list with annotations for + # for each normal argument. If an argument doesn't have an + # annotation, its value will be None. + # pylint: disable=too-many-instance-attributes + _astroid_fields = ( + "args", + "defaults", + "kwonlyargs", + "posonlyargs", + "posonlyargs_annotations", + "kw_defaults", + "annotations", + "varargannotation", + "kwargannotation", + "kwonlyargs_annotations", + "type_comment_args", + "type_comment_kwonlyargs", + "type_comment_posonlyargs", + ) + varargannotation = None + """The type annotation for the variable length arguments. + + :type: NodeNG + """ + kwargannotation = None + """The type annotation for the variable length keyword arguments. + + :type: NodeNG + """ + + _other_fields = ("vararg", "kwarg") + + def __init__(self, vararg=None, kwarg=None, parent=None): + """ + :param vararg: The name of the variable length arguments. + :type vararg: str or None + + :param kwarg: The name of the variable length keyword arguments. + :type kwarg: str or None + + :param parent: The parent node in the syntax tree. + :type parent: NodeNG or None + """ + super().__init__(parent=parent) + self.vararg = vararg + """The name of the variable length arguments. + + :type: str or None + """ + + self.kwarg = kwarg + """The name of the variable length keyword arguments. + + :type: str or None + """ + + self.args = [] + """The names of the required arguments. + + :type: list(AssignName) + """ + + self.defaults = [] + """The default values for arguments that can be passed positionally. + + :type: list(NodeNG) + """ + + self.kwonlyargs = [] + """The keyword arguments that cannot be passed positionally. + + :type: list(AssignName) + """ + + self.posonlyargs = [] + """The arguments that can only be passed positionally. + + :type: list(AssignName) + """ + + self.kw_defaults = [] + """The default values for keyword arguments that cannot be passed positionally. + + :type: list(NodeNG) + """ + + self.annotations = [] + """The type annotations of arguments that can be passed positionally. + + :type: list(NodeNG) + """ + + self.posonlyargs_annotations = [] + """The type annotations of arguments that can only be passed positionally. + + :type: list(NodeNG) + """ + + self.kwonlyargs_annotations = [] + """The type annotations of arguments that cannot be passed positionally. + + :type: list(NodeNG) + """ + + self.type_comment_args = [] + """The type annotation, passed by a type comment, of each argument. + + If an argument does not have a type comment, + the value for that argument will be None. + + :type: list(NodeNG or None) + """ + + self.type_comment_kwonlyargs = [] + """The type annotation, passed by a type comment, of each keyword only argument. + + If an argument does not have a type comment, + the value for that argument will be None. + + :type: list(NodeNG or None) + """ + + self.type_comment_posonlyargs = [] + """The type annotation, passed by a type comment, of each positional argument. + + If an argument does not have a type comment, + the value for that argument will be None. + + :type: list(NodeNG or None) + """ + + # pylint: disable=too-many-arguments + def postinit( + self, + args, + defaults, + kwonlyargs, + kw_defaults, + annotations, + posonlyargs=None, + kwonlyargs_annotations=None, + posonlyargs_annotations=None, + varargannotation=None, + kwargannotation=None, + type_comment_args=None, + type_comment_kwonlyargs=None, + type_comment_posonlyargs=None, + ): + """Do some setup after initialisation. + + :param args: The names of the required arguments. + :type args: list(AssignName) + + :param defaults: The default values for arguments that can be passed + positionally. + :type defaults: list(NodeNG) + + :param kwonlyargs: The keyword arguments that cannot be passed + positionally. + :type kwonlyargs: list(AssignName) + + :param posonlyargs: The arguments that can only be passed + positionally. + :type kwonlyargs: list(AssignName) + + :param kw_defaults: The default values for keyword arguments that + cannot be passed positionally. + :type kw_defaults: list(NodeNG) + + :param annotations: The type annotations of arguments that can be + passed positionally. + :type annotations: list(NodeNG) + + :param kwonlyargs_annotations: The type annotations of arguments that + cannot be passed positionally. This should always be passed in + Python 3. + :type kwonlyargs_annotations: list(NodeNG) + + :param posonlyargs_annotations: The type annotations of arguments that + can only be passed positionally. This should always be passed in + Python 3. + :type posonlyargs_annotations: list(NodeNG) + + :param varargannotation: The type annotation for the variable length + arguments. + :type varargannotation: NodeNG + + :param kwargannotation: The type annotation for the variable length + keyword arguments. + :type kwargannotation: NodeNG + + :param type_comment_args: The type annotation, + passed by a type comment, of each argument. + :type type_comment_args: list(NodeNG or None) + + :param type_comment_args: The type annotation, + passed by a type comment, of each keyword only argument. + :type type_comment_args: list(NodeNG or None) + + :param type_comment_args: The type annotation, + passed by a type comment, of each positional argument. + :type type_comment_args: list(NodeNG or None) + """ + self.args = args + self.defaults = defaults + self.kwonlyargs = kwonlyargs + self.posonlyargs = posonlyargs + self.kw_defaults = kw_defaults + self.annotations = annotations + self.kwonlyargs_annotations = kwonlyargs_annotations + self.posonlyargs_annotations = posonlyargs_annotations + self.varargannotation = varargannotation + self.kwargannotation = kwargannotation + self.type_comment_args = type_comment_args + self.type_comment_kwonlyargs = type_comment_kwonlyargs + self.type_comment_posonlyargs = type_comment_posonlyargs + + # pylint: disable=too-many-arguments + + def _infer_name(self, frame, name): + if self.parent is frame: + return name + return None + + @decorators.cachedproperty + def fromlineno(self): + """The first line that this node appears on in the source code. + + :type: int or None + """ + lineno = super().fromlineno + return max(lineno, self.parent.fromlineno or 0) + + @decorators.cachedproperty + def arguments(self): + """Get all the arguments for this node, including positional only and positional and keyword""" + return list(itertools.chain((self.posonlyargs or ()), self.args or ())) + + def format_args(self): + """Get the arguments formatted as string. + + :returns: The formatted arguments. + :rtype: str + """ + result = [] + positional_only_defaults = [] + positional_or_keyword_defaults = self.defaults + if self.defaults: + args = self.args or [] + positional_or_keyword_defaults = self.defaults[-len(args) :] + positional_only_defaults = self.defaults[: len(self.defaults) - len(args)] + + if self.posonlyargs: + result.append( + _format_args( + self.posonlyargs, + positional_only_defaults, + self.posonlyargs_annotations, + ) + ) + result.append("/") + if self.args: + result.append( + _format_args( + self.args, + positional_or_keyword_defaults, + getattr(self, "annotations", None), + ) + ) + if self.vararg: + result.append("*%s" % self.vararg) + if self.kwonlyargs: + if not self.vararg: + result.append("*") + result.append( + _format_args( + self.kwonlyargs, self.kw_defaults, self.kwonlyargs_annotations + ) + ) + if self.kwarg: + result.append("**%s" % self.kwarg) + return ", ".join(result) + + def default_value(self, argname): + """Get the default value for an argument. + + :param argname: The name of the argument to get the default value for. + :type argname: str + + :raises NoDefault: If there is no default value defined for the + given argument. + """ + args = self.arguments + index = _find_arg(argname, args)[0] + if index is not None: + idx = index - (len(args) - len(self.defaults)) + if idx >= 0: + return self.defaults[idx] + index = _find_arg(argname, self.kwonlyargs)[0] + if index is not None and self.kw_defaults[index] is not None: + return self.kw_defaults[index] + raise exceptions.NoDefault(func=self.parent, name=argname) + + def is_argument(self, name): + """Check if the given name is defined in the arguments. + + :param name: The name to check for. + :type name: str + + :returns: True if the given name is defined in the arguments, + False otherwise. + :rtype: bool + """ + if name == self.vararg: + return True + if name == self.kwarg: + return True + return ( + self.find_argname(name, rec=True)[1] is not None + or self.kwonlyargs + and _find_arg(name, self.kwonlyargs, rec=True)[1] is not None + ) + + def find_argname(self, argname, rec=False): + """Get the index and :class:`AssignName` node for given name. + + :param argname: The name of the argument to search for. + :type argname: str + + :param rec: Whether or not to include arguments in unpacked tuples + in the search. + :type rec: bool + + :returns: The index and node for the argument. + :rtype: tuple(str or None, AssignName or None) + """ + if self.arguments: + return _find_arg(argname, self.arguments, rec) + return None, None + + def get_children(self): + yield from self.posonlyargs or () + + for elt in self.posonlyargs_annotations: + if elt is not None: + yield elt + + yield from self.args or () + + yield from self.defaults + yield from self.kwonlyargs + + for elt in self.kw_defaults: + if elt is not None: + yield elt + + for elt in self.annotations: + if elt is not None: + yield elt + + if self.varargannotation is not None: + yield self.varargannotation + + if self.kwargannotation is not None: + yield self.kwargannotation + + for elt in self.kwonlyargs_annotations: + if elt is not None: + yield elt + + +def _find_arg(argname, args, rec=False): + for i, arg in enumerate(args): + if isinstance(arg, Tuple): + if rec: + found = _find_arg(argname, arg.elts) + if found[0] is not None: + return found + elif arg.name == argname: + return i, arg + return None, None + + +def _format_args(args, defaults=None, annotations=None): + values = [] + if args is None: + return "" + if annotations is None: + annotations = [] + if defaults is not None: + default_offset = len(args) - len(defaults) + packed = itertools.zip_longest(args, annotations) + for i, (arg, annotation) in enumerate(packed): + if isinstance(arg, Tuple): + values.append("(%s)" % _format_args(arg.elts)) + else: + argname = arg.name + default_sep = "=" + if annotation is not None: + argname += ": " + annotation.as_string() + default_sep = " = " + values.append(argname) + + if defaults is not None and i >= default_offset: + if defaults[i - default_offset] is not None: + values[-1] += default_sep + defaults[i - default_offset].as_string() + return ", ".join(values) + + +class AssignAttr(mixins.ParentAssignTypeMixin, NodeNG): + """Variation of :class:`ast.Assign` representing assignment to an attribute. + + >>> node = astroid.extract_node('self.attribute = range(10)') + >>> node + + >>> list(node.get_children()) + [, ] + >>> list(node.get_children())[0].as_string() + 'self.attribute' + """ + + _astroid_fields = ("expr",) + _other_fields = ("attrname",) + expr = None + """What has the attribute that is being assigned to. + + :type: NodeNG or None + """ + + def __init__(self, attrname=None, lineno=None, col_offset=None, parent=None): + """ + :param attrname: The name of the attribute being assigned to. + :type attrname: str or None + + :param lineno: The line that this node appears on in the source code. + :type lineno: int or None + + :param col_offset: The column that this node appears on in the + source code. + :type col_offset: int or None + + :param parent: The parent node in the syntax tree. + :type parent: NodeNG or None + """ + self.attrname = attrname + """The name of the attribute being assigned to. + + :type: str or None + """ + + super().__init__(lineno, col_offset, parent) + + def postinit(self, expr=None): + """Do some setup after initialisation. + + :param expr: What has the attribute that is being assigned to. + :type expr: NodeNG or None + """ + self.expr = expr + + def get_children(self): + yield self.expr + + +class Assert(Statement): + """Class representing an :class:`ast.Assert` node. + + An :class:`Assert` node represents an assert statement. + + >>> node = astroid.extract_node('assert len(things) == 10, "Not enough things"') + >>> node + + """ + + _astroid_fields = ("test", "fail") + test = None + """The test that passes or fails the assertion. + + :type: NodeNG or None + """ + fail = None + """The message shown when the assertion fails. + + :type: NodeNG or None + """ + + def postinit(self, test=None, fail=None): + """Do some setup after initialisation. + + :param test: The test that passes or fails the assertion. + :type test: NodeNG or None + + :param fail: The message shown when the assertion fails. + :type fail: NodeNG or None + """ + self.fail = fail + self.test = test + + def get_children(self): + yield self.test + + if self.fail is not None: + yield self.fail + + +class Assign(mixins.AssignTypeMixin, Statement): + """Class representing an :class:`ast.Assign` node. + + An :class:`Assign` is a statement where something is explicitly + asssigned to. + + >>> node = astroid.extract_node('variable = range(10)') + >>> node + + """ + + _astroid_fields = ("targets", "value") + _other_other_fields = ("type_annotation",) + targets = None + """What is being assigned to. + + :type: list(NodeNG) or None + """ + value = None + """The value being assigned to the variables. + + :type: NodeNG or None + """ + type_annotation = None + """If present, this will contain the type annotation passed by a type comment + + :type: NodeNG or None + """ + + def postinit(self, targets=None, value=None, type_annotation=None): + """Do some setup after initialisation. + + :param targets: What is being assigned to. + :type targets: list(NodeNG) or None + + :param value: The value being assigned to the variables. + :type: NodeNG or None + """ + self.targets = targets + self.value = value + self.type_annotation = type_annotation + + def get_children(self): + yield from self.targets + + yield self.value + + @decorators.cached + def _get_assign_nodes(self): + return [self] + list(self.value._get_assign_nodes()) + + def _get_yield_nodes_skip_lambdas(self): + yield from self.value._get_yield_nodes_skip_lambdas() + + +class AnnAssign(mixins.AssignTypeMixin, Statement): + """Class representing an :class:`ast.AnnAssign` node. + + An :class:`AnnAssign` is an assignment with a type annotation. + + >>> node = astroid.extract_node('variable: List[int] = range(10)') + >>> node + + """ + + _astroid_fields = ("target", "annotation", "value") + _other_fields = ("simple",) + target = None + """What is being assigned to. + + :type: NodeNG or None + """ + annotation = None + """The type annotation of what is being assigned to. + + :type: NodeNG + """ + value = None + """The value being assigned to the variables. + + :type: NodeNG or None + """ + simple = None + """Whether :attr:`target` is a pure name or a complex statement. + + :type: int + """ + + def postinit(self, target, annotation, simple, value=None): + """Do some setup after initialisation. + + :param target: What is being assigned to. + :type target: NodeNG + + :param annotation: The type annotation of what is being assigned to. + :type: NodeNG + + :param simple: Whether :attr:`target` is a pure name + or a complex statement. + :type simple: int + + :param value: The value being assigned to the variables. + :type: NodeNG or None + """ + self.target = target + self.annotation = annotation + self.value = value + self.simple = simple + + def get_children(self): + yield self.target + yield self.annotation + + if self.value is not None: + yield self.value + + +class AugAssign(mixins.AssignTypeMixin, Statement): + """Class representing an :class:`ast.AugAssign` node. + + An :class:`AugAssign` is an assignment paired with an operator. + + >>> node = astroid.extract_node('variable += 1') + >>> node + + """ + + _astroid_fields = ("target", "value") + _other_fields = ("op",) + target = None + """What is being assigned to. + + :type: NodeNG or None + """ + value = None + """The value being assigned to the variable. + + :type: NodeNG or None + """ + + def __init__(self, op=None, lineno=None, col_offset=None, parent=None): + """ + :param op: The operator that is being combined with the assignment. + This includes the equals sign. + :type op: str or None + + :param lineno: The line that this node appears on in the source code. + :type lineno: int or None + + :param col_offset: The column that this node appears on in the + source code. + :type col_offset: int or None + + :param parent: The parent node in the syntax tree. + :type parent: NodeNG or None + """ + self.op = op + """The operator that is being combined with the assignment. + + This includes the equals sign. + + :type: str or None + """ + + super().__init__(lineno, col_offset, parent) + + def postinit(self, target=None, value=None): + """Do some setup after initialisation. + + :param target: What is being assigned to. + :type target: NodeNG or None + + :param value: The value being assigned to the variable. + :type: NodeNG or None + """ + self.target = target + self.value = value + + # This is set by inference.py + def _infer_augassign(self, context=None): + raise NotImplementedError + + def type_errors(self, context=None): + """Get a list of type errors which can occur during inference. + + Each TypeError is represented by a :class:`BadBinaryOperationMessage` , + which holds the original exception. + + :returns: The list of possible type errors. + :rtype: list(BadBinaryOperationMessage) + """ + try: + results = self._infer_augassign(context=context) + return [ + result + for result in results + if isinstance(result, util.BadBinaryOperationMessage) + ] + except exceptions.InferenceError: + return [] + + def get_children(self): + yield self.target + yield self.value + + +class Repr(NodeNG): + """Class representing an :class:`ast.Repr` node. + + A :class:`Repr` node represents the backtick syntax, + which is a deprecated alias for :func:`repr` removed in Python 3. + + >>> node = astroid.extract_node('`variable`') + >>> node + + """ + + _astroid_fields = ("value",) + value = None + """What is having :func:`repr` called on it. + + :type: NodeNG or None + """ + + def postinit(self, value=None): + """Do some setup after initialisation. + + :param value: What is having :func:`repr` called on it. + :type value: NodeNG or None + """ + self.value = value + + +class BinOp(NodeNG): + """Class representing an :class:`ast.BinOp` node. + + A :class:`BinOp` node is an application of a binary operator. + + >>> node = astroid.extract_node('a + b') + >>> node + + """ + + _astroid_fields = ("left", "right") + _other_fields = ("op",) + left = None + """What is being applied to the operator on the left side. + + :type: NodeNG or None + """ + right = None + """What is being applied to the operator on the right side. + + :type: NodeNG or None + """ + + def __init__(self, op=None, lineno=None, col_offset=None, parent=None): + """ + :param op: The operator. + :type: str or None + + :param lineno: The line that this node appears on in the source code. + :type lineno: int or None + + :param col_offset: The column that this node appears on in the + source code. + :type col_offset: int or None + + :param parent: The parent node in the syntax tree. + :type parent: NodeNG or None + """ + self.op = op + """The operator. + + :type: str or None + """ + + super().__init__(lineno, col_offset, parent) + + def postinit(self, left=None, right=None): + """Do some setup after initialisation. + + :param left: What is being applied to the operator on the left side. + :type left: NodeNG or None + + :param right: What is being applied to the operator on the right side. + :type right: NodeNG or None + """ + self.left = left + self.right = right + + # This is set by inference.py + def _infer_binop(self, context=None): + raise NotImplementedError + + def type_errors(self, context=None): + """Get a list of type errors which can occur during inference. + + Each TypeError is represented by a :class:`BadBinaryOperationMessage`, + which holds the original exception. + + :returns: The list of possible type errors. + :rtype: list(BadBinaryOperationMessage) + """ + try: + results = self._infer_binop(context=context) + return [ + result + for result in results + if isinstance(result, util.BadBinaryOperationMessage) + ] + except exceptions.InferenceError: + return [] + + def get_children(self): + yield self.left + yield self.right + + def op_precedence(self): + return OP_PRECEDENCE[self.op] + + def op_left_associative(self): + # 2**3**4 == 2**(3**4) + return self.op != "**" + + +class BoolOp(NodeNG): + """Class representing an :class:`ast.BoolOp` node. + + A :class:`BoolOp` is an application of a boolean operator. + + >>> node = astroid.extract_node('a and b') + >>> node + + """ + + _astroid_fields = ("values",) + _other_fields = ("op",) + values = None + """The values being applied to the operator. + + :type: list(NodeNG) or None + """ + + def __init__(self, op=None, lineno=None, col_offset=None, parent=None): + """ + :param op: The operator. + :type: str or None + + :param lineno: The line that this node appears on in the source code. + :type lineno: int or None + + :param col_offset: The column that this node appears on in the + source code. + :type col_offset: int or None + + :param parent: The parent node in the syntax tree. + :type parent: NodeNG or None + """ + self.op = op + """The operator. + + :type: str or None + """ + + super().__init__(lineno, col_offset, parent) + + def postinit(self, values=None): + """Do some setup after initialisation. + + :param values: The values being applied to the operator. + :type values: list(NodeNG) or None + """ + self.values = values + + def get_children(self): + yield from self.values + + def op_precedence(self): + return OP_PRECEDENCE[self.op] + + +class Break(mixins.NoChildrenMixin, Statement): + """Class representing an :class:`ast.Break` node. + + >>> node = astroid.extract_node('break') + >>> node + + """ + + +class Call(NodeNG): + """Class representing an :class:`ast.Call` node. + + A :class:`Call` node is a call to a function, method, etc. + + >>> node = astroid.extract_node('function()') + >>> node + + """ + + _astroid_fields = ("func", "args", "keywords") + func = None + """What is being called. + + :type: NodeNG or None + """ + args = None + """The positional arguments being given to the call. + + :type: list(NodeNG) or None + """ + keywords = None + """The keyword arguments being given to the call. + + :type: list(NodeNG) or None + """ + + def postinit(self, func=None, args=None, keywords=None): + """Do some setup after initialisation. + + :param func: What is being called. + :type func: NodeNG or None + + :param args: The positional arguments being given to the call. + :type args: list(NodeNG) or None + + :param keywords: The keyword arguments being given to the call. + :type keywords: list(NodeNG) or None + """ + self.func = func + self.args = args + self.keywords = keywords + + @property + def starargs(self): + """The positional arguments that unpack something. + + :type: list(Starred) + """ + args = self.args or [] + return [arg for arg in args if isinstance(arg, Starred)] + + @property + def kwargs(self): + """The keyword arguments that unpack something. + + :type: list(Keyword) + """ + keywords = self.keywords or [] + return [keyword for keyword in keywords if keyword.arg is None] + + def get_children(self): + yield self.func + + yield from self.args + + yield from self.keywords or () + + +class Compare(NodeNG): + """Class representing an :class:`ast.Compare` node. + + A :class:`Compare` node indicates a comparison. + + >>> node = astroid.extract_node('a <= b <= c') + >>> node + + >>> node.ops + [('<=', ), ('<=', )] + """ + + _astroid_fields = ("left", "ops") + left = None + """The value at the left being applied to a comparison operator. + + :type: NodeNG or None + """ + ops = None + """The remainder of the operators and their relevant right hand value. + + :type: list(tuple(str, NodeNG)) or None + """ + + def postinit(self, left=None, ops=None): + """Do some setup after initialisation. + + :param left: The value at the left being applied to a comparison + operator. + :type left: NodeNG or None + + :param ops: The remainder of the operators + and their relevant right hand value. + :type ops: list(tuple(str, NodeNG)) or None + """ + self.left = left + self.ops = ops + + def get_children(self): + """Get the child nodes below this node. + + Overridden to handle the tuple fields and skip returning the operator + strings. + + :returns: The children. + :rtype: iterable(NodeNG) + """ + yield self.left + for _, comparator in self.ops: + yield comparator # we don't want the 'op' + + def last_child(self): + """An optimized version of list(get_children())[-1] + + :returns: The last child. + :rtype: NodeNG + """ + # XXX maybe if self.ops: + return self.ops[-1][1] + # return self.left + + +class Comprehension(NodeNG): + """Class representing an :class:`ast.comprehension` node. + + A :class:`Comprehension` indicates the loop inside any type of + comprehension including generator expressions. + + >>> node = astroid.extract_node('[x for x in some_values]') + >>> list(node.get_children()) + [, ] + >>> list(node.get_children())[1].as_string() + 'for x in some_values' + """ + + _astroid_fields = ("target", "iter", "ifs") + _other_fields = ("is_async",) + target = None + """What is assigned to by the comprehension. + + :type: NodeNG or None + """ + iter = None + """What is iterated over by the comprehension. + + :type: NodeNG or None + """ + ifs = None + """The contents of any if statements that filter the comprehension. + + :type: list(NodeNG) or None + """ + is_async = None + """Whether this is an asynchronous comprehension or not. + + :type: bool or None + """ + + def __init__(self, parent=None): + """ + :param parent: The parent node in the syntax tree. + :type parent: NodeNG or None + """ + super().__init__() + self.parent = parent + + # pylint: disable=redefined-builtin; same name as builtin ast module. + def postinit(self, target=None, iter=None, ifs=None, is_async=None): + """Do some setup after initialisation. + + :param target: What is assigned to by the comprehension. + :type target: NodeNG or None + + :param iter: What is iterated over by the comprehension. + :type iter: NodeNG or None + + :param ifs: The contents of any if statements that filter + the comprehension. + :type ifs: list(NodeNG) or None + + :param is_async: Whether this is an asynchronous comprehension or not. + :type: bool or None + """ + self.target = target + self.iter = iter + self.ifs = ifs + self.is_async = is_async + + optional_assign = True + """Whether this node optionally assigns a variable. + + :type: bool + """ + + def assign_type(self): + """The type of assignment that this node performs. + + :returns: The assignment type. + :rtype: NodeNG + """ + return self + + def _get_filtered_stmts(self, lookup_node, node, stmts, mystmt): + """method used in filter_stmts""" + if self is mystmt: + if isinstance(lookup_node, (Const, Name)): + return [lookup_node], True + + elif self.statement() is mystmt: + # original node's statement is the assignment, only keeps + # current node (gen exp, list comp) + + return [node], True + + return stmts, False + + def get_children(self): + yield self.target + yield self.iter + + yield from self.ifs + + +class Const(mixins.NoChildrenMixin, NodeNG, bases.Instance): + """Class representing any constant including num, str, bool, None, bytes. + + >>> node = astroid.extract_node('(5, "This is a string.", True, None, b"bytes")') + >>> node + + >>> list(node.get_children()) + [, + , + , + , + ] + """ + + _other_fields = ("value",) + + def __init__(self, value, lineno=None, col_offset=None, parent=None): + """ + :param value: The value that the constant represents. + :type value: object + + :param lineno: The line that this node appears on in the source code. + :type lineno: int or None + + :param col_offset: The column that this node appears on in the + source code. + :type col_offset: int or None + + :param parent: The parent node in the syntax tree. + :type parent: NodeNG or None + """ + self.value = value + """The value that the constant represents. + + :type: object + """ + + super().__init__(lineno, col_offset, parent) + + def __getattr__(self, name): + # This is needed because of Proxy's __getattr__ method. + # Calling object.__new__ on this class without calling + # __init__ would result in an infinite loop otherwise + # since __getattr__ is called when an attribute doesn't + # exist and self._proxied indirectly calls self.value + # and Proxy __getattr__ calls self.value + if name == "value": + raise AttributeError + return super().__getattr__(name) + + def getitem(self, index, context=None): + """Get an item from this node if subscriptable. + + :param index: The node to use as a subscript index. + :type index: Const or Slice + + :raises AstroidTypeError: When the given index cannot be used as a + subscript index, or if this node is not subscriptable. + """ + if isinstance(index, Const): + index_value = index.value + elif isinstance(index, Slice): + index_value = _infer_slice(index, context=context) + + else: + raise exceptions.AstroidTypeError( + "Could not use type {} as subscript index".format(type(index)) + ) + + try: + if isinstance(self.value, (str, bytes)): + return Const(self.value[index_value]) + except IndexError as exc: + raise exceptions.AstroidIndexError( + message="Index {index!r} out of range", + node=self, + index=index, + context=context, + ) from exc + except TypeError as exc: + raise exceptions.AstroidTypeError( + message="Type error {error!r}", node=self, index=index, context=context + ) from exc + + raise exceptions.AstroidTypeError("%r (value=%s)" % (self, self.value)) + + def has_dynamic_getattr(self): + """Check if the node has a custom __getattr__ or __getattribute__. + + :returns: True if the class has a custom + __getattr__ or __getattribute__, False otherwise. + For a :class:`Const` this is always ``False``. + :rtype: bool + """ + return False + + def itered(self): + """An iterator over the elements this node contains. + + :returns: The contents of this node. + :rtype: iterable(Const) + + :raises TypeError: If this node does not represent something that is iterable. + """ + if isinstance(self.value, str): + return [const_factory(elem) for elem in self.value] + raise TypeError("Cannot iterate over type {!r}".format(type(self.value))) + + def pytype(self): + """Get the name of the type that this node represents. + + :returns: The name of the type. + :rtype: str + """ + return self._proxied.qname() + + def bool_value(self, context=None): + """Determine the boolean value of this node. + + :returns: The boolean value of this node. + :rtype: bool + """ + return bool(self.value) + + +class Continue(mixins.NoChildrenMixin, Statement): + """Class representing an :class:`ast.Continue` node. + + >>> node = astroid.extract_node('continue') + >>> node + + """ + + +class Decorators(NodeNG): + """A node representing a list of decorators. + + A :class:`Decorators` is the decorators that are applied to + a method or function. + + >>> node = astroid.extract_node(''' + @property + def my_property(self): + return 3 + ''') + >>> node + + >>> list(node.get_children())[0] + + """ + + _astroid_fields = ("nodes",) + nodes = None + """The decorators that this node contains. + + :type: list(Name or Call) or None + """ + + def postinit(self, nodes): + """Do some setup after initialisation. + + :param nodes: The decorators that this node contains. + :type nodes: list(Name or Call) + """ + self.nodes = nodes + + def scope(self): + """The first parent node defining a new scope. + + :returns: The first parent scope node. + :rtype: Module or FunctionDef or ClassDef or Lambda or GenExpr + """ + # skip the function node to go directly to the upper level scope + return self.parent.parent.scope() + + def get_children(self): + yield from self.nodes + + +class DelAttr(mixins.ParentAssignTypeMixin, NodeNG): + """Variation of :class:`ast.Delete` representing deletion of an attribute. + + >>> node = astroid.extract_node('del self.attr') + >>> node + + >>> list(node.get_children())[0] + + """ + + _astroid_fields = ("expr",) + _other_fields = ("attrname",) + expr = None + """The name that this node represents. + + :type: Name or None + """ + + def __init__(self, attrname=None, lineno=None, col_offset=None, parent=None): + """ + :param attrname: The name of the attribute that is being deleted. + :type attrname: str or None + + :param lineno: The line that this node appears on in the source code. + :type lineno: int or None + + :param col_offset: The column that this node appears on in the + source code. + :type col_offset: int or None + + :param parent: The parent node in the syntax tree. + :type parent: NodeNG or None + """ + self.attrname = attrname + """The name of the attribute that is being deleted. + + :type: str or None + """ + + super().__init__(lineno, col_offset, parent) + + def postinit(self, expr=None): + """Do some setup after initialisation. + + :param expr: The name that this node represents. + :type expr: Name or None + """ + self.expr = expr + + def get_children(self): + yield self.expr + + +class Delete(mixins.AssignTypeMixin, Statement): + """Class representing an :class:`ast.Delete` node. + + A :class:`Delete` is a ``del`` statement this is deleting something. + + >>> node = astroid.extract_node('del self.attr') + >>> node + + """ + + _astroid_fields = ("targets",) + targets = None + """What is being deleted. + + :type: list(NodeNG) or None + """ + + def postinit(self, targets=None): + """Do some setup after initialisation. + + :param targets: What is being deleted. + :type targets: list(NodeNG) or None + """ + self.targets = targets + + def get_children(self): + yield from self.targets + + +class Dict(NodeNG, bases.Instance): + """Class representing an :class:`ast.Dict` node. + + A :class:`Dict` is a dictionary that is created with ``{}`` syntax. + + >>> node = astroid.extract_node('{1: "1"}') + >>> node + + """ + + _astroid_fields = ("items",) + + def __init__(self, lineno=None, col_offset=None, parent=None): + """ + :param lineno: The line that this node appears on in the source code. + :type lineno: int or None + + :param col_offset: The column that this node appears on in the + source code. + :type col_offset: int or None + + :param parent: The parent node in the syntax tree. + :type parent: NodeNG or None + """ + self.items = [] + """The key-value pairs contained in the dictionary. + + :type: list(tuple(NodeNG, NodeNG)) + """ + + super().__init__(lineno, col_offset, parent) + + def postinit(self, items): + """Do some setup after initialisation. + + :param items: The key-value pairs contained in the dictionary. + :type items: list(tuple(NodeNG, NodeNG)) + """ + self.items = items + + @classmethod + def from_elements(cls, items=None): + """Create a :class:`Dict` of constants from a live dictionary. + + :param items: The items to store in the node. + :type items: dict + + :returns: The created dictionary node. + :rtype: Dict + """ + node = cls() + if items is None: + node.items = [] + else: + node.items = [ + (const_factory(k), const_factory(v) if _is_const(v) else v) + for k, v in items.items() + # The keys need to be constants + if _is_const(k) + ] + return node + + def pytype(self): + """Get the name of the type that this node represents. + + :returns: The name of the type. + :rtype: str + """ + return "%s.dict" % BUILTINS + + def get_children(self): + """Get the key and value nodes below this node. + + Children are returned in the order that they are defined in the source + code, key first then the value. + + :returns: The children. + :rtype: iterable(NodeNG) + """ + for key, value in self.items: + yield key + yield value + + def last_child(self): + """An optimized version of list(get_children())[-1] + + :returns: The last child, or None if no children exist. + :rtype: NodeNG or None + """ + if self.items: + return self.items[-1][1] + return None + + def itered(self): + """An iterator over the keys this node contains. + + :returns: The keys of this node. + :rtype: iterable(NodeNG) + """ + return [key for (key, _) in self.items] + + def getitem(self, index, context=None): + """Get an item from this node. + + :param index: The node to use as a subscript index. + :type index: Const or Slice + + :raises AstroidTypeError: When the given index cannot be used as a + subscript index, or if this node is not subscriptable. + :raises AstroidIndexError: If the given index does not exist in the + dictionary. + """ + for key, value in self.items: + # TODO(cpopa): no support for overriding yet, {1:2, **{1: 3}}. + if isinstance(key, DictUnpack): + try: + return value.getitem(index, context) + except (exceptions.AstroidTypeError, exceptions.AstroidIndexError): + continue + for inferredkey in key.infer(context): + if inferredkey is util.Uninferable: + continue + if isinstance(inferredkey, Const) and isinstance(index, Const): + if inferredkey.value == index.value: + return value + + raise exceptions.AstroidIndexError(index) + + def bool_value(self, context=None): + """Determine the boolean value of this node. + + :returns: The boolean value of this node. + :rtype: bool + """ + return bool(self.items) + + +class Expr(Statement): + """Class representing an :class:`ast.Expr` node. + + An :class:`Expr` is any expression that does not have its value used or + stored. + + >>> node = astroid.extract_node('method()') + >>> node + + >>> node.parent + + """ + + _astroid_fields = ("value",) + value = None + """What the expression does. + + :type: NodeNG or None + """ + + def postinit(self, value=None): + """Do some setup after initialisation. + + :param value: What the expression does. + :type value: NodeNG or None + """ + self.value = value + + def get_children(self): + yield self.value + + def _get_yield_nodes_skip_lambdas(self): + if not self.value.is_lambda: + yield from self.value._get_yield_nodes_skip_lambdas() + + +class Ellipsis(mixins.NoChildrenMixin, NodeNG): # pylint: disable=redefined-builtin + """Class representing an :class:`ast.Ellipsis` node. + + An :class:`Ellipsis` is the ``...`` syntax. + + >>> node = astroid.extract_node('...') + >>> node + + """ + + def bool_value(self, context=None): + """Determine the boolean value of this node. + + :returns: The boolean value of this node. + For an :class:`Ellipsis` this is always ``True``. + :rtype: bool + """ + return True + + +class EmptyNode(mixins.NoChildrenMixin, NodeNG): + """Holds an arbitrary object in the :attr:`LocalsDictNodeNG.locals`.""" + + object = None + + +class ExceptHandler(mixins.MultiLineBlockMixin, mixins.AssignTypeMixin, Statement): + """Class representing an :class:`ast.ExceptHandler`. node. + + An :class:`ExceptHandler` is an ``except`` block on a try-except. + + >>> node = astroid.extract_node(''' + try: + do_something() + except Exception as error: + print("Error!") + ''') + >>> node + + >>> >>> node.handlers + [] + """ + + _astroid_fields = ("type", "name", "body") + _multi_line_block_fields = ("body",) + type = None + """The types that the block handles. + + :type: Tuple or NodeNG or None + """ + name = None + """The name that the caught exception is assigned to. + + :type: AssignName or None + """ + body = None + """The contents of the block. + + :type: list(NodeNG) or None + """ + + def get_children(self): + if self.type is not None: + yield self.type + + if self.name is not None: + yield self.name + + yield from self.body + + # pylint: disable=redefined-builtin; had to use the same name as builtin ast module. + def postinit(self, type=None, name=None, body=None): + """Do some setup after initialisation. + + :param type: The types that the block handles. + :type type: Tuple or NodeNG or None + + :param name: The name that the caught exception is assigned to. + :type name: AssignName or None + + :param body:The contents of the block. + :type body: list(NodeNG) or None + """ + self.type = type + self.name = name + self.body = body + + @decorators.cachedproperty + def blockstart_tolineno(self): + """The line on which the beginning of this block ends. + + :type: int + """ + if self.name: + return self.name.tolineno + if self.type: + return self.type.tolineno + return self.lineno + + def catch(self, exceptions): # pylint: disable=redefined-outer-name + """Check if this node handles any of the given exceptions. + + If ``exceptions`` is empty, this will default to ``True``. + + :param exceptions: The name of the exceptions to check for. + :type exceptions: list(str) + """ + if self.type is None or exceptions is None: + return True + for node in self.type._get_name_nodes(): + if node.name in exceptions: + return True + return False + + +class Exec(Statement): + """Class representing the ``exec`` statement. + + >>> node = astroid.extract_node('exec "True"') + >>> node + + """ + + _astroid_fields = ("expr", "globals", "locals") + expr = None + """The expression to be executed. + + :type: NodeNG or None + """ + globals = None + """The globals dictionary to execute with. + + :type: NodeNG or None + """ + locals = None + """The locals dictionary to execute with. + + :type: NodeNG or None + """ + + # pylint: disable=redefined-builtin; had to use the same name as builtin ast module. + def postinit(self, expr=None, globals=None, locals=None): + """Do some setup after initialisation. + + :param expr: The expression to be executed. + :type expr: NodeNG or None + + :param globals:The globals dictionary to execute with. + :type globals: NodeNG or None + + :param locals: The locals dictionary to execute with. + :type locals: NodeNG or None + """ + self.expr = expr + self.globals = globals + self.locals = locals + + +class ExtSlice(NodeNG): + """Class representing an :class:`ast.ExtSlice` node. + + An :class:`ExtSlice` is a complex slice expression. + + >>> node = astroid.extract_node('l[1:3, 5]') + >>> node + + >>> node.slice + + """ + + _astroid_fields = ("dims",) + dims = None + """The simple dimensions that form the complete slice. + + :type: list(NodeNG) or None + """ + + def postinit(self, dims=None): + """Do some setup after initialisation. + + :param dims: The simple dimensions that form the complete slice. + :type dims: list(NodeNG) or None + """ + self.dims = dims + + +class For( + mixins.MultiLineBlockMixin, + mixins.BlockRangeMixIn, + mixins.AssignTypeMixin, + Statement, +): + """Class representing an :class:`ast.For` node. + + >>> node = astroid.extract_node('for thing in things: print(thing)') + >>> node + + """ + + _astroid_fields = ("target", "iter", "body", "orelse") + _other_other_fields = ("type_annotation",) + _multi_line_block_fields = ("body", "orelse") + target = None + """What the loop assigns to. + + :type: NodeNG or None + """ + iter = None + """What the loop iterates over. + + :type: NodeNG or None + """ + body = None + """The contents of the body of the loop. + + :type: list(NodeNG) or None + """ + orelse = None + """The contents of the ``else`` block of the loop. + + :type: list(NodeNG) or None + """ + type_annotation = None + """If present, this will contain the type annotation passed by a type comment + + :type: NodeNG or None + """ + + # pylint: disable=redefined-builtin; had to use the same name as builtin ast module. + def postinit( + self, target=None, iter=None, body=None, orelse=None, type_annotation=None + ): + """Do some setup after initialisation. + + :param target: What the loop assigns to. + :type target: NodeNG or None + + :param iter: What the loop iterates over. + :type iter: NodeNG or None + + :param body: The contents of the body of the loop. + :type body: list(NodeNG) or None + + :param orelse: The contents of the ``else`` block of the loop. + :type orelse: list(NodeNG) or None + """ + self.target = target + self.iter = iter + self.body = body + self.orelse = orelse + self.type_annotation = type_annotation + + optional_assign = True + """Whether this node optionally assigns a variable. + + This is always ``True`` for :class:`For` nodes. + + :type: bool + """ + + @decorators.cachedproperty + def blockstart_tolineno(self): + """The line on which the beginning of this block ends. + + :type: int + """ + return self.iter.tolineno + + def get_children(self): + yield self.target + yield self.iter + + yield from self.body + yield from self.orelse + + +class AsyncFor(For): + """Class representing an :class:`ast.AsyncFor` node. + + An :class:`AsyncFor` is an asynchronous :class:`For` built with + the ``async`` keyword. + + >>> node = astroid.extract_node(''' + async def func(things): + async for thing in things: + print(thing) + ''') + >>> node + + >>> node.body[0] + + """ + + +class Await(NodeNG): + """Class representing an :class:`ast.Await` node. + + An :class:`Await` is the ``await`` keyword. + + >>> node = astroid.extract_node(''' + async def func(things): + await other_func() + ''') + >>> node + + >>> node.body[0] + + >>> list(node.body[0].get_children())[0] + + """ + + _astroid_fields = ("value",) + value = None + """What to wait for. + + :type: NodeNG or None + """ + + def postinit(self, value=None): + """Do some setup after initialisation. + + :param value: What to wait for. + :type value: NodeNG or None + """ + self.value = value + + def get_children(self): + yield self.value + + +class ImportFrom(mixins.NoChildrenMixin, mixins.ImportFromMixin, Statement): + """Class representing an :class:`ast.ImportFrom` node. + + >>> node = astroid.extract_node('from my_package import my_module') + >>> node + + """ + + _other_fields = ("modname", "names", "level") + + def __init__( + self, fromname, names, level=0, lineno=None, col_offset=None, parent=None + ): + """ + :param fromname: The module that is being imported from. + :type fromname: str or None + + :param names: What is being imported from the module. + :type names: list(tuple(str, str or None)) + + :param level: The level of relative import. + :type level: int + + :param lineno: The line that this node appears on in the source code. + :type lineno: int or None + + :param col_offset: The column that this node appears on in the + source code. + :type col_offset: int or None + + :param parent: The parent node in the syntax tree. + :type parent: NodeNG or None + """ + self.modname = fromname + """The module that is being imported from. + + This is ``None`` for relative imports. + + :type: str or None + """ + + self.names = names + """What is being imported from the module. + + Each entry is a :class:`tuple` of the name being imported, + and the alias that the name is assigned to (if any). + + :type: list(tuple(str, str or None)) + """ + + self.level = level + """The level of relative import. + + Essentially this is the number of dots in the import. + This is always 0 for absolute imports. + + :type: int + """ + + super().__init__(lineno, col_offset, parent) + + +class Attribute(NodeNG): + """Class representing an :class:`ast.Attribute` node.""" + + _astroid_fields = ("expr",) + _other_fields = ("attrname",) + expr = None + """The name that this node represents. + + :type: Name or None + """ + + def __init__(self, attrname=None, lineno=None, col_offset=None, parent=None): + """ + :param attrname: The name of the attribute. + :type attrname: str or None + + :param lineno: The line that this node appears on in the source code. + :type lineno: int or None + + :param col_offset: The column that this node appears on in the + source code. + :type col_offset: int or None + + :param parent: The parent node in the syntax tree. + :type parent: NodeNG or None + """ + self.attrname = attrname + """The name of the attribute. + + :type: str or None + """ + + super().__init__(lineno, col_offset, parent) + + def postinit(self, expr=None): + """Do some setup after initialisation. + + :param expr: The name that this node represents. + :type expr: Name or None + """ + self.expr = expr + + def get_children(self): + yield self.expr + + +class Global(mixins.NoChildrenMixin, Statement): + """Class representing an :class:`ast.Global` node. + + >>> node = astroid.extract_node('global a_global') + >>> node + + """ + + _other_fields = ("names",) + + def __init__(self, names, lineno=None, col_offset=None, parent=None): + """ + :param names: The names being declared as global. + :type names: list(str) + + :param lineno: The line that this node appears on in the source code. + :type lineno: int or None + + :param col_offset: The column that this node appears on in the + source code. + :type col_offset: int or None + + :param parent: The parent node in the syntax tree. + :type parent: NodeNG or None + """ + self.names = names + """The names being declared as global. + + :type: list(str) + """ + + super().__init__(lineno, col_offset, parent) + + def _infer_name(self, frame, name): + return name + + +class If(mixins.MultiLineBlockMixin, mixins.BlockRangeMixIn, Statement): + """Class representing an :class:`ast.If` node. + + >>> node = astroid.extract_node('if condition: print(True)') + >>> node + + """ + + _astroid_fields = ("test", "body", "orelse") + _multi_line_block_fields = ("body", "orelse") + test = None + """The condition that the statement tests. + + :type: NodeNG or None + """ + body = None + """The contents of the block. + + :type: list(NodeNG) or None + """ + orelse = None + """The contents of the ``else`` block. + + :type: list(NodeNG) or None + """ + + def postinit(self, test=None, body=None, orelse=None): + """Do some setup after initialisation. + + :param test: The condition that the statement tests. + :type test: NodeNG or None + + :param body: The contents of the block. + :type body: list(NodeNG) or None + + :param orelse: The contents of the ``else`` block. + :type orelse: list(NodeNG) or None + """ + self.test = test + self.body = body + self.orelse = orelse + + @decorators.cachedproperty + def blockstart_tolineno(self): + """The line on which the beginning of this block ends. + + :type: int + """ + return self.test.tolineno + + def block_range(self, lineno): + """Get a range from the given line number to where this node ends. + + :param lineno: The line number to start the range at. + :type lineno: int + + :returns: The range of line numbers that this node belongs to, + starting at the given line number. + :rtype: tuple(int, int) + """ + if lineno == self.body[0].fromlineno: + return lineno, lineno + if lineno <= self.body[-1].tolineno: + return lineno, self.body[-1].tolineno + return self._elsed_block_range(lineno, self.orelse, self.body[0].fromlineno - 1) + + def get_children(self): + yield self.test + + yield from self.body + yield from self.orelse + + def has_elif_block(self): + return len(self.orelse) == 1 and isinstance(self.orelse[0], If) + + +class IfExp(NodeNG): + """Class representing an :class:`ast.IfExp` node. + + >>> node = astroid.extract_node('value if condition else other') + >>> node + + """ + + _astroid_fields = ("test", "body", "orelse") + test = None + """The condition that the statement tests. + + :type: NodeNG or None + """ + body = None + """The contents of the block. + + :type: list(NodeNG) or None + """ + orelse = None + """The contents of the ``else`` block. + + :type: list(NodeNG) or None + """ + + def postinit(self, test=None, body=None, orelse=None): + """Do some setup after initialisation. + + :param test: The condition that the statement tests. + :type test: NodeNG or None + + :param body: The contents of the block. + :type body: list(NodeNG) or None + + :param orelse: The contents of the ``else`` block. + :type orelse: list(NodeNG) or None + """ + self.test = test + self.body = body + self.orelse = orelse + + def get_children(self): + yield self.test + yield self.body + yield self.orelse + + def op_left_associative(self): + # `1 if True else 2 if False else 3` is parsed as + # `1 if True else (2 if False else 3)` + return False + + +class Import(mixins.NoChildrenMixin, mixins.ImportFromMixin, Statement): + """Class representing an :class:`ast.Import` node. + + >>> node = astroid.extract_node('import astroid') + >>> node + + """ + + _other_fields = ("names",) + + def __init__(self, names=None, lineno=None, col_offset=None, parent=None): + """ + :param names: The names being imported. + :type names: list(tuple(str, str or None)) or None + + :param lineno: The line that this node appears on in the source code. + :type lineno: int or None + + :param col_offset: The column that this node appears on in the + source code. + :type col_offset: int or None + + :param parent: The parent node in the syntax tree. + :type parent: NodeNG or None + """ + self.names = names + """The names being imported. + + Each entry is a :class:`tuple` of the name being imported, + and the alias that the name is assigned to (if any). + + :type: list(tuple(str, str or None)) or None + """ + + super().__init__(lineno, col_offset, parent) + + +class Index(NodeNG): + """Class representing an :class:`ast.Index` node. + + An :class:`Index` is a simple subscript. + + >>> node = astroid.extract_node('things[1]') + >>> node + + >>> node.slice + + """ + + _astroid_fields = ("value",) + value = None + """The value to subscript with. + + :type: NodeNG or None + """ + + def postinit(self, value=None): + """Do some setup after initialisation. + + :param value: The value to subscript with. + :type value: NodeNG or None + """ + self.value = value + + def get_children(self): + yield self.value + + +class Keyword(NodeNG): + """Class representing an :class:`ast.keyword` node. + + >>> node = astroid.extract_node('function(a_kwarg=True)') + >>> node + + >>> node.keywords + [] + """ + + _astroid_fields = ("value",) + _other_fields = ("arg",) + value = None + """The value being assigned to the keyword argument. + + :type: NodeNG or None + """ + + def __init__(self, arg=None, lineno=None, col_offset=None, parent=None): + """ + :param arg: The argument being assigned to. + :type arg: Name or None + + :param lineno: The line that this node appears on in the source code. + :type lineno: int or None + + :param col_offset: The column that this node appears on in the + source code. + :type col_offset: int or None + + :param parent: The parent node in the syntax tree. + :type parent: NodeNG or None + """ + self.arg = arg + """The argument being assigned to. + + :type: Name or None + """ + + super().__init__(lineno, col_offset, parent) + + def postinit(self, value=None): + """Do some setup after initialisation. + + :param value: The value being assigned to the ketword argument. + :type value: NodeNG or None + """ + self.value = value + + def get_children(self): + yield self.value + + +class List(_BaseContainer): + """Class representing an :class:`ast.List` node. + + >>> node = astroid.extract_node('[1, 2, 3]') + >>> node + + """ + + _other_fields = ("ctx",) + + def __init__(self, ctx=None, lineno=None, col_offset=None, parent=None): + """ + :param ctx: Whether the list is assigned to or loaded from. + :type ctx: Context or None + + :param lineno: The line that this node appears on in the source code. + :type lineno: int or None + + :param col_offset: The column that this node appears on in the + source code. + :type col_offset: int or None + + :param parent: The parent node in the syntax tree. + :type parent: NodeNG or None + """ + self.ctx = ctx + """Whether the list is assigned to or loaded from. + + :type: Context or None + """ + + super().__init__(lineno, col_offset, parent) + + def pytype(self): + """Get the name of the type that this node represents. + + :returns: The name of the type. + :rtype: str + """ + return "%s.list" % BUILTINS + + def getitem(self, index, context=None): + """Get an item from this node. + + :param index: The node to use as a subscript index. + :type index: Const or Slice + """ + return _container_getitem(self, self.elts, index, context=context) + + +class Nonlocal(mixins.NoChildrenMixin, Statement): + """Class representing an :class:`ast.Nonlocal` node. + + >>> node = astroid.extract_node(''' + def function(): + nonlocal var + ''') + >>> node + + >>> node.body[0] + + """ + + _other_fields = ("names",) + + def __init__(self, names, lineno=None, col_offset=None, parent=None): + """ + :param names: The names being declared as not local. + :type names: list(str) + + :param lineno: The line that this node appears on in the source code. + :type lineno: int or None + + :param col_offset: The column that this node appears on in the + source code. + :type col_offset: int or None + + :param parent: The parent node in the syntax tree. + :type parent: NodeNG or None + """ + self.names = names + """The names being declared as not local. + + :type: list(str) + """ + + super().__init__(lineno, col_offset, parent) + + def _infer_name(self, frame, name): + return name + + +class Pass(mixins.NoChildrenMixin, Statement): + """Class representing an :class:`ast.Pass` node. + + >>> node = astroid.extract_node('pass') + >>> node + + """ + + +class Print(Statement): + """Class representing an :class:`ast.Print` node. + + >>> node = astroid.extract_node('print "A message"') + >>> node + + """ + + _astroid_fields = ("dest", "values") + dest = None + """Where to print to. + + :type: NodeNG or None + """ + values = None + """What to print. + + :type: list(NodeNG) or None + """ + + def __init__(self, nl=None, lineno=None, col_offset=None, parent=None): + """ + :param nl: Whether to print a new line. + :type nl: bool or None + + :param lineno: The line that this node appears on in the source code. + :type lineno: int or None + + :param col_offset: The column that this node appears on in the + source code. + :type col_offset: int or None + + :param parent: The parent node in the syntax tree. + :type parent: NodeNG or None + """ + self.nl = nl + """Whether to print a new line. + + :type: bool or None + """ + + super().__init__(lineno, col_offset, parent) + + def postinit(self, dest=None, values=None): + """Do some setup after initialisation. + + :param dest: Where to print to. + :type dest: NodeNG or None + + :param values: What to print. + :type values: list(NodeNG) or None + """ + self.dest = dest + self.values = values + + +class Raise(Statement): + """Class representing an :class:`ast.Raise` node. + + >>> node = astroid.extract_node('raise RuntimeError("Something bad happened!")') + >>> node + + """ + + exc = None + """What is being raised. + + :type: NodeNG or None + """ + _astroid_fields = ("exc", "cause") + cause = None + """The exception being used to raise this one. + + :type: NodeNG or None + """ + + def postinit(self, exc=None, cause=None): + """Do some setup after initialisation. + + :param exc: What is being raised. + :type exc: NodeNG or None + + :param cause: The exception being used to raise this one. + :type cause: NodeNG or None + """ + self.exc = exc + self.cause = cause + + def raises_not_implemented(self): + """Check if this node raises a :class:`NotImplementedError`. + + :returns: True if this node raises a :class:`NotImplementedError`, + False otherwise. + :rtype: bool + """ + if not self.exc: + return False + for name in self.exc._get_name_nodes(): + if name.name == "NotImplementedError": + return True + return False + + def get_children(self): + if self.exc is not None: + yield self.exc + + if self.cause is not None: + yield self.cause + + +class Return(Statement): + """Class representing an :class:`ast.Return` node. + + >>> node = astroid.extract_node('return True') + >>> node + + """ + + _astroid_fields = ("value",) + value = None + """The value being returned. + + :type: NodeNG or None + """ + + def postinit(self, value=None): + """Do some setup after initialisation. + + :param value: The value being returned. + :type value: NodeNG or None + """ + self.value = value + + def get_children(self): + if self.value is not None: + yield self.value + + def is_tuple_return(self): + return isinstance(self.value, Tuple) + + def _get_return_nodes_skip_functions(self): + yield self + + +class Set(_BaseContainer): + """Class representing an :class:`ast.Set` node. + + >>> node = astroid.extract_node('{1, 2, 3}') + >>> node + + """ + + def pytype(self): + """Get the name of the type that this node represents. + + :returns: The name of the type. + :rtype: str + """ + return "%s.set" % BUILTINS + + +class Slice(NodeNG): + """Class representing an :class:`ast.Slice` node. + + >>> node = astroid.extract_node('things[1:3]') + >>> node + + >>> node.slice + + """ + + _astroid_fields = ("lower", "upper", "step") + lower = None + """The lower index in the slice. + + :type: NodeNG or None + """ + upper = None + """The upper index in the slice. + + :type: NodeNG or None + """ + step = None + """The step to take between indexes. + + :type: NodeNG or None + """ + + def postinit(self, lower=None, upper=None, step=None): + """Do some setup after initialisation. + + :param lower: The lower index in the slice. + :value lower: NodeNG or None + + :param upper: The upper index in the slice. + :value upper: NodeNG or None + + :param step: The step to take between index. + :param step: NodeNG or None + """ + self.lower = lower + self.upper = upper + self.step = step + + def _wrap_attribute(self, attr): + """Wrap the empty attributes of the Slice in a Const node.""" + if not attr: + const = const_factory(attr) + const.parent = self + return const + return attr + + @decorators.cachedproperty + def _proxied(self): + builtins = MANAGER.builtins_module + return builtins.getattr("slice")[0] + + def pytype(self): + """Get the name of the type that this node represents. + + :returns: The name of the type. + :rtype: str + """ + return "%s.slice" % BUILTINS + + def igetattr(self, attrname, context=None): + """Infer the possible values of the given attribute on the slice. + + :param attrname: The name of the attribute to infer. + :type attrname: str + + :returns: The inferred possible values. + :rtype: iterable(NodeNG) + """ + if attrname == "start": + yield self._wrap_attribute(self.lower) + elif attrname == "stop": + yield self._wrap_attribute(self.upper) + elif attrname == "step": + yield self._wrap_attribute(self.step) + else: + yield from self.getattr(attrname, context=context) + + def getattr(self, attrname, context=None): + return self._proxied.getattr(attrname, context) + + def get_children(self): + if self.lower is not None: + yield self.lower + + if self.upper is not None: + yield self.upper + + if self.step is not None: + yield self.step + + +class Starred(mixins.ParentAssignTypeMixin, NodeNG): + """Class representing an :class:`ast.Starred` node. + + >>> node = astroid.extract_node('*args') + >>> node + + """ + + _astroid_fields = ("value",) + _other_fields = ("ctx",) + value = None + """What is being unpacked. + + :type: NodeNG or None + """ + + def __init__(self, ctx=None, lineno=None, col_offset=None, parent=None): + """ + :param ctx: Whether the list is assigned to or loaded from. + :type ctx: Context or None + + :param lineno: The line that this node appears on in the source code. + :type lineno: int or None + + :param col_offset: The column that this node appears on in the + source code. + :type col_offset: int or None + + :param parent: The parent node in the syntax tree. + :type parent: NodeNG or None + """ + self.ctx = ctx + """Whether the starred item is assigned to or loaded from. + + :type: Context or None + """ + + super().__init__(lineno=lineno, col_offset=col_offset, parent=parent) + + def postinit(self, value=None): + """Do some setup after initialisation. + + :param value: What is being unpacked. + :type value: NodeNG or None + """ + self.value = value + + def get_children(self): + yield self.value + + +class Subscript(NodeNG): + """Class representing an :class:`ast.Subscript` node. + + >>> node = astroid.extract_node('things[1:3]') + >>> node + + """ + + _astroid_fields = ("value", "slice") + _other_fields = ("ctx",) + value = None + """What is being indexed. + + :type: NodeNG or None + """ + slice = None + """The slice being used to lookup. + + :type: NodeNG or None + """ + + def __init__(self, ctx=None, lineno=None, col_offset=None, parent=None): + """ + :param ctx: Whether the subscripted item is assigned to or loaded from. + :type ctx: Context or None + + :param lineno: The line that this node appears on in the source code. + :type lineno: int or None + + :param col_offset: The column that this node appears on in the + source code. + :type col_offset: int or None + + :param parent: The parent node in the syntax tree. + :type parent: NodeNG or None + """ + self.ctx = ctx + """Whether the subscripted item is assigned to or loaded from. + + :type: Context or None + """ + + super().__init__(lineno=lineno, col_offset=col_offset, parent=parent) + + # pylint: disable=redefined-builtin; had to use the same name as builtin ast module. + def postinit(self, value=None, slice=None): + """Do some setup after initialisation. + + :param value: What is being indexed. + :type value: NodeNG or None + + :param slice: The slice being used to lookup. + :type slice: NodeNG or None + """ + self.value = value + self.slice = slice + + def get_children(self): + yield self.value + yield self.slice + + +class TryExcept(mixins.MultiLineBlockMixin, mixins.BlockRangeMixIn, Statement): + """Class representing an :class:`ast.TryExcept` node. + + >>> node = astroid.extract_node(''' + try: + do_something() + except Exception as error: + print("Error!") + ''') + >>> node + + """ + + _astroid_fields = ("body", "handlers", "orelse") + _multi_line_block_fields = ("body", "handlers", "orelse") + body = None + """The contents of the block to catch exceptions from. + + :type: list(NodeNG) or None + """ + handlers = None + """The exception handlers. + + :type: list(ExceptHandler) or None + """ + orelse = None + """The contents of the ``else`` block. + + :type: list(NodeNG) or None + """ + + def postinit(self, body=None, handlers=None, orelse=None): + """Do some setup after initialisation. + + :param body: The contents of the block to catch exceptions from. + :type body: list(NodeNG) or None + + :param handlers: The exception handlers. + :type handlers: list(ExceptHandler) or None + + :param orelse: The contents of the ``else`` block. + :type orelse: list(NodeNG) or None + """ + self.body = body + self.handlers = handlers + self.orelse = orelse + + def _infer_name(self, frame, name): + return name + + def block_range(self, lineno): + """Get a range from the given line number to where this node ends. + + :param lineno: The line number to start the range at. + :type lineno: int + + :returns: The range of line numbers that this node belongs to, + starting at the given line number. + :rtype: tuple(int, int) + """ + last = None + for exhandler in self.handlers: + if exhandler.type and lineno == exhandler.type.fromlineno: + return lineno, lineno + if exhandler.body[0].fromlineno <= lineno <= exhandler.body[-1].tolineno: + return lineno, exhandler.body[-1].tolineno + if last is None: + last = exhandler.body[0].fromlineno - 1 + return self._elsed_block_range(lineno, self.orelse, last) + + def get_children(self): + yield from self.body + + yield from self.handlers or () + yield from self.orelse or () + + +class TryFinally(mixins.MultiLineBlockMixin, mixins.BlockRangeMixIn, Statement): + """Class representing an :class:`ast.TryFinally` node. + + >>> node = astroid.extract_node(''' + try: + do_something() + except Exception as error: + print("Error!") + finally: + print("Cleanup!") + ''') + >>> node + + """ + + _astroid_fields = ("body", "finalbody") + _multi_line_block_fields = ("body", "finalbody") + body = None + """The try-except that the finally is attached to. + + :type: list(TryExcept) or None + """ + finalbody = None + """The contents of the ``finally`` block. + + :type: list(NodeNG) or None + """ + + def postinit(self, body=None, finalbody=None): + """Do some setup after initialisation. + + :param body: The try-except that the finally is attached to. + :type body: list(TryExcept) or None + + :param finalbody: The contents of the ``finally`` block. + :type finalbody: list(NodeNG) or None + """ + self.body = body + self.finalbody = finalbody + + def block_range(self, lineno): + """Get a range from the given line number to where this node ends. + + :param lineno: The line number to start the range at. + :type lineno: int + + :returns: The range of line numbers that this node belongs to, + starting at the given line number. + :rtype: tuple(int, int) + """ + child = self.body[0] + # py2.5 try: except: finally: + if ( + isinstance(child, TryExcept) + and child.fromlineno == self.fromlineno + and child.tolineno >= lineno > self.fromlineno + ): + return child.block_range(lineno) + return self._elsed_block_range(lineno, self.finalbody) + + def get_children(self): + yield from self.body + yield from self.finalbody + + +class Tuple(_BaseContainer): + """Class representing an :class:`ast.Tuple` node. + + >>> node = astroid.extract_node('(1, 2, 3)') + >>> node + + """ + + _other_fields = ("ctx",) + + def __init__(self, ctx=None, lineno=None, col_offset=None, parent=None): + """ + :param ctx: Whether the tuple is assigned to or loaded from. + :type ctx: Context or None + + :param lineno: The line that this node appears on in the source code. + :type lineno: int or None + + :param col_offset: The column that this node appears on in the + source code. + :type col_offset: int or None + + :param parent: The parent node in the syntax tree. + :type parent: NodeNG or None + """ + self.ctx = ctx + """Whether the tuple is assigned to or loaded from. + + :type: Context or None + """ + + super().__init__(lineno, col_offset, parent) + + def pytype(self): + """Get the name of the type that this node represents. + + :returns: The name of the type. + :rtype: str + """ + return "%s.tuple" % BUILTINS + + def getitem(self, index, context=None): + """Get an item from this node. + + :param index: The node to use as a subscript index. + :type index: Const or Slice + """ + return _container_getitem(self, self.elts, index, context=context) + + +class UnaryOp(NodeNG): + """Class representing an :class:`ast.UnaryOp` node. + + >>> node = astroid.extract_node('-5') + >>> node + + """ + + _astroid_fields = ("operand",) + _other_fields = ("op",) + operand = None + """What the unary operator is applied to. + + :type: NodeNG or None + """ + + def __init__(self, op=None, lineno=None, col_offset=None, parent=None): + """ + :param op: The operator. + :type: str or None + + :param lineno: The line that this node appears on in the source code. + :type lineno: int or None + + :param col_offset: The column that this node appears on in the + source code. + :type col_offset: int or None + + :param parent: The parent node in the syntax tree. + :type parent: NodeNG or None + """ + self.op = op + """The operator. + + :type: str or None + """ + + super().__init__(lineno, col_offset, parent) + + def postinit(self, operand=None): + """Do some setup after initialisation. + + :param operand: What the unary operator is applied to. + :type operand: NodeNG or None + """ + self.operand = operand + + # This is set by inference.py + def _infer_unaryop(self, context=None): + raise NotImplementedError + + def type_errors(self, context=None): + """Get a list of type errors which can occur during inference. + + Each TypeError is represented by a :class:`BadBinaryOperationMessage`, + which holds the original exception. + + :returns: The list of possible type errors. + :rtype: list(BadBinaryOperationMessage) + """ + try: + results = self._infer_unaryop(context=context) + return [ + result + for result in results + if isinstance(result, util.BadUnaryOperationMessage) + ] + except exceptions.InferenceError: + return [] + + def get_children(self): + yield self.operand + + def op_precedence(self): + if self.op == "not": + return OP_PRECEDENCE[self.op] + + return super().op_precedence() + + +class While(mixins.MultiLineBlockMixin, mixins.BlockRangeMixIn, Statement): + """Class representing an :class:`ast.While` node. + + >>> node = astroid.extract_node(''' + while condition(): + print("True") + ''') + >>> node + + """ + + _astroid_fields = ("test", "body", "orelse") + _multi_line_block_fields = ("body", "orelse") + test = None + """The condition that the loop tests. + + :type: NodeNG or None + """ + body = None + """The contents of the loop. + + :type: list(NodeNG) or None + """ + orelse = None + """The contents of the ``else`` block. + + :type: list(NodeNG) or None + """ + + def postinit(self, test=None, body=None, orelse=None): + """Do some setup after initialisation. + + :param test: The condition that the loop tests. + :type test: NodeNG or None + + :param body: The contents of the loop. + :type body: list(NodeNG) or None + + :param orelse: The contents of the ``else`` block. + :type orelse: list(NodeNG) or None + """ + self.test = test + self.body = body + self.orelse = orelse + + @decorators.cachedproperty + def blockstart_tolineno(self): + """The line on which the beginning of this block ends. + + :type: int + """ + return self.test.tolineno + + def block_range(self, lineno): + """Get a range from the given line number to where this node ends. + + :param lineno: The line number to start the range at. + :type lineno: int + + :returns: The range of line numbers that this node belongs to, + starting at the given line number. + :rtype: tuple(int, int) + """ + return self._elsed_block_range(lineno, self.orelse) + + def get_children(self): + yield self.test + + yield from self.body + yield from self.orelse + + def _get_yield_nodes_skip_lambdas(self): + """A While node can contain a Yield node in the test""" + yield from self.test._get_yield_nodes_skip_lambdas() + yield from super()._get_yield_nodes_skip_lambdas() + + +class With( + mixins.MultiLineBlockMixin, + mixins.BlockRangeMixIn, + mixins.AssignTypeMixin, + Statement, +): + """Class representing an :class:`ast.With` node. + + >>> node = astroid.extract_node(''' + with open(file_path) as file_: + print(file_.read()) + ''') + >>> node + + """ + + _astroid_fields = ("items", "body") + _other_other_fields = ("type_annotation",) + _multi_line_block_fields = ("body",) + items = None + """The pairs of context managers and the names they are assigned to. + + :type: list(tuple(NodeNG, AssignName or None)) or None + """ + body = None + """The contents of the ``with`` block. + + :type: list(NodeNG) or None + """ + type_annotation = None + """If present, this will contain the type annotation passed by a type comment + + :type: NodeNG or None + """ + + def postinit(self, items=None, body=None, type_annotation=None): + """Do some setup after initialisation. + + :param items: The pairs of context managers and the names + they are assigned to. + :type items: list(tuple(NodeNG, AssignName or None)) or None + + :param body: The contents of the ``with`` block. + :type body: list(NodeNG) or None + """ + self.items = items + self.body = body + self.type_annotation = type_annotation + + @decorators.cachedproperty + def blockstart_tolineno(self): + """The line on which the beginning of this block ends. + + :type: int + """ + return self.items[-1][0].tolineno + + def get_children(self): + """Get the child nodes below this node. + + :returns: The children. + :rtype: iterable(NodeNG) + """ + for expr, var in self.items: + yield expr + if var: + yield var + yield from self.body + + +class AsyncWith(With): + """Asynchronous ``with`` built with the ``async`` keyword.""" + + +class Yield(NodeNG): + """Class representing an :class:`ast.Yield` node. + + >>> node = astroid.extract_node('yield True') + >>> node + + """ + + _astroid_fields = ("value",) + value = None + """The value to yield. + + :type: NodeNG or None + """ + + def postinit(self, value=None): + """Do some setup after initialisation. + + :param value: The value to yield. + :type value: NodeNG or None + """ + self.value = value + + def get_children(self): + if self.value is not None: + yield self.value + + def _get_yield_nodes_skip_lambdas(self): + yield self + + +class YieldFrom(Yield): + """Class representing an :class:`ast.YieldFrom` node.""" + + +class DictUnpack(mixins.NoChildrenMixin, NodeNG): + """Represents the unpacking of dicts into dicts using :pep:`448`.""" + + +class FormattedValue(NodeNG): + """Class representing an :class:`ast.FormattedValue` node. + + Represents a :pep:`498` format string. + + >>> node = astroid.extract_node('f"Format {type_}"') + >>> node + + >>> node.values + [, ] + """ + + _astroid_fields = ("value", "format_spec") + value = None + """The value to be formatted into the string. + + :type: NodeNG or None + """ + conversion = None + """The type of formatting to be applied to the value. + + .. seealso:: + :class:`ast.FormattedValue` + + :type: int or None + """ + format_spec = None + """The formatting to be applied to the value. + + .. seealso:: + :class:`ast.FormattedValue` + + :type: JoinedStr or None + """ + + def postinit(self, value, conversion=None, format_spec=None): + """Do some setup after initialisation. + + :param value: The value to be formatted into the string. + :type value: NodeNG + + :param conversion: The type of formatting to be applied to the value. + :type conversion: int or None + + :param format_spec: The formatting to be applied to the value. + :type format_spec: JoinedStr or None + """ + self.value = value + self.conversion = conversion + self.format_spec = format_spec + + def get_children(self): + yield self.value + + if self.format_spec is not None: + yield self.format_spec + + +class JoinedStr(NodeNG): + """Represents a list of string expressions to be joined. + + >>> node = astroid.extract_node('f"Format {type_}"') + >>> node + + """ + + _astroid_fields = ("values",) + values = None + """The string expressions to be joined. + + :type: list(FormattedValue or Const) or None + """ + + def postinit(self, values=None): + """Do some setup after initialisation. + + :param value: The string expressions to be joined. + + :type: list(FormattedValue or Const) or None + """ + self.values = values + + def get_children(self): + yield from self.values + + +class NamedExpr(mixins.AssignTypeMixin, NodeNG): + """Represents the assignment from the assignment expression + + >>> module = astroid.parse('if a := 1: pass') + >>> module.body[0].test + + """ + + _astroid_fields = ("target", "value") + target = None + """The assignment target + + :type: Name + """ + value = None + """The value that gets assigned in the expression + + :type: NodeNG + """ + + def postinit(self, target, value): + self.target = target + self.value = value + + +class Unknown(mixins.AssignTypeMixin, NodeNG): + """This node represents a node in a constructed AST where + introspection is not possible. At the moment, it's only used in + the args attribute of FunctionDef nodes where function signature + introspection failed. + """ + + name = "Unknown" + + def qname(self): + return "Unknown" + + def infer(self, context=None, **kwargs): + """Inference on an Unknown node immediately terminates.""" + yield util.Uninferable + + +class EvaluatedObject(NodeNG): + """Contains an object that has already been inferred + + This class is useful to pre-evaluate a particular node, + with the resulting class acting as the non-evaluated node. + """ + + name = "EvaluatedObject" + _astroid_fields = ("original",) + _other_fields = ("value",) + + original = None + """The original node that has already been evaluated + + :type: NodeNG + """ + + value = None + """The inferred value + + :type: Union[Uninferable, NodeNG] + """ + + def __init__(self, original, value): + self.original = original + self.value = value + super().__init__( + lineno=self.original.lineno, + col_offset=self.original.col_offset, + parent=self.original.parent, + ) + + def infer(self, context=None, **kwargs): + yield self.value + + +# constants ############################################################## + +CONST_CLS = { + list: List, + tuple: Tuple, + dict: Dict, + set: Set, + type(None): Const, + type(NotImplemented): Const, +} +if PY38: + CONST_CLS[type(...)] = Const + + +def _update_const_classes(): + """update constant classes, so the keys of CONST_CLS can be reused""" + klasses = (bool, int, float, complex, str, bytes) + for kls in klasses: + CONST_CLS[kls] = Const + + +_update_const_classes() + + +def _two_step_initialization(cls, value): + instance = cls() + instance.postinit(value) + return instance + + +def _dict_initialization(cls, value): + if isinstance(value, dict): + value = tuple(value.items()) + return _two_step_initialization(cls, value) + + +_CONST_CLS_CONSTRUCTORS = { + List: _two_step_initialization, + Tuple: _two_step_initialization, + Dict: _dict_initialization, + Set: _two_step_initialization, + Const: lambda cls, value: cls(value), +} + + +def const_factory(value): + """return an astroid node for a python value""" + # XXX we should probably be stricter here and only consider stuff in + # CONST_CLS or do better treatment: in case where value is not in CONST_CLS, + # we should rather recall the builder on this value than returning an empty + # node (another option being that const_factory shouldn't be called with something + # not in CONST_CLS) + assert not isinstance(value, NodeNG) + + # Hack for ignoring elements of a sequence + # or a mapping, in order to avoid transforming + # each element to an AST. This is fixed in 2.0 + # and this approach is a temporary hack. + if isinstance(value, (list, set, tuple, dict)): + elts = [] + else: + elts = value + + try: + initializer_cls = CONST_CLS[value.__class__] + initializer = _CONST_CLS_CONSTRUCTORS[initializer_cls] + return initializer(initializer_cls, elts) + except (KeyError, AttributeError): + node = EmptyNode() + node.object = value + return node + + +def is_from_decorator(node): + """Return True if the given node is the child of a decorator""" + parent = node.parent + while parent is not None: + if isinstance(parent, Decorators): + return True + parent = parent.parent + return False diff --git a/WebCrawler/venv/Lib/site-packages/astroid/nodes.py b/WebCrawler/venv/Lib/site-packages/astroid/nodes.py new file mode 100644 index 0000000..4ce4ebe --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/astroid/nodes.py @@ -0,0 +1,176 @@ +# Copyright (c) 2006-2011, 2013 LOGILAB S.A. (Paris, FRANCE) +# Copyright (c) 2010 Daniel Harding +# Copyright (c) 2014-2020 Claudiu Popa +# Copyright (c) 2014 Google, Inc. +# Copyright (c) 2015-2016 Ceridwen +# Copyright (c) 2016 Jared Garst +# Copyright (c) 2017 Ashley Whetter +# Copyright (c) 2017 rr- +# Copyright (c) 2018 Bryce Guinta + +# Licensed under the LGPL: https://www.gnu.org/licenses/old-licenses/lgpl-2.1.en.html +# For details: https://github.com/PyCQA/astroid/blob/master/COPYING.LESSER + +"""Every available node class. + +.. seealso:: + :doc:`ast documentation ` + +All nodes inherit from :class:`~astroid.node_classes.NodeNG`. +""" +# pylint: disable=unused-import,redefined-builtin + +from astroid.node_classes import ( + Arguments, + AssignAttr, + Assert, + Assign, + AnnAssign, + AssignName, + AugAssign, + Repr, + BinOp, + BoolOp, + Break, + Call, + Compare, + Comprehension, + Const, + Continue, + Decorators, + DelAttr, + DelName, + Delete, + Dict, + Expr, + Ellipsis, + EmptyNode, + ExceptHandler, + Exec, + ExtSlice, + For, + ImportFrom, + Attribute, + Global, + If, + IfExp, + Import, + Index, + Keyword, + List, + Name, + NamedExpr, + Nonlocal, + Pass, + Print, + Raise, + Return, + Set, + Slice, + Starred, + Subscript, + TryExcept, + TryFinally, + Tuple, + UnaryOp, + While, + With, + Yield, + YieldFrom, + const_factory, + AsyncFor, + Await, + AsyncWith, + FormattedValue, + JoinedStr, + # Node not present in the builtin ast module. + DictUnpack, + Unknown, + EvaluatedObject, +) +from astroid.scoped_nodes import ( + Module, + GeneratorExp, + Lambda, + DictComp, + ListComp, + SetComp, + FunctionDef, + ClassDef, + AsyncFunctionDef, +) + + +ALL_NODE_CLASSES = ( + AsyncFunctionDef, + AsyncFor, + AsyncWith, + Await, + Arguments, + AssignAttr, + Assert, + Assign, + AnnAssign, + AssignName, + AugAssign, + Repr, + BinOp, + BoolOp, + Break, + Call, + ClassDef, + Compare, + Comprehension, + Const, + Continue, + Decorators, + DelAttr, + DelName, + Delete, + Dict, + DictComp, + DictUnpack, + Expr, + Ellipsis, + EmptyNode, + ExceptHandler, + Exec, + ExtSlice, + For, + ImportFrom, + FunctionDef, + Attribute, + GeneratorExp, + Global, + If, + IfExp, + Import, + Index, + Keyword, + Lambda, + List, + ListComp, + Name, + NamedExpr, + Nonlocal, + Module, + Pass, + Print, + Raise, + Return, + Set, + SetComp, + Slice, + Starred, + Subscript, + TryExcept, + TryFinally, + Tuple, + UnaryOp, + While, + With, + Yield, + YieldFrom, + FormattedValue, + JoinedStr, +) diff --git a/WebCrawler/venv/Lib/site-packages/astroid/objects.py b/WebCrawler/venv/Lib/site-packages/astroid/objects.py new file mode 100644 index 0000000..fb782e6 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/astroid/objects.py @@ -0,0 +1,314 @@ +# Copyright (c) 2015-2016, 2018-2020 Claudiu Popa +# Copyright (c) 2015-2016 Ceridwen +# Copyright (c) 2015 Florian Bruhin +# Copyright (c) 2016 Derek Gustafson +# Copyright (c) 2018 hippo91 +# Copyright (c) 2018 Bryce Guinta + +# Licensed under the LGPL: https://www.gnu.org/licenses/old-licenses/lgpl-2.1.en.html +# For details: https://github.com/PyCQA/astroid/blob/master/COPYING.LESSER + + +""" +Inference objects are a way to represent composite AST nodes, +which are used only as inference results, so they can't be found in the +original AST tree. For instance, inferring the following frozenset use, +leads to an inferred FrozenSet: + + Call(func=Name('frozenset'), args=Tuple(...)) +""" + +import builtins + +from astroid import bases +from astroid import decorators +from astroid import exceptions +from astroid import MANAGER +from astroid import node_classes +from astroid import scoped_nodes +from astroid import util + + +BUILTINS = builtins.__name__ +objectmodel = util.lazy_import("interpreter.objectmodel") + + +class FrozenSet(node_classes._BaseContainer): + """class representing a FrozenSet composite node""" + + def pytype(self): + return "%s.frozenset" % BUILTINS + + def _infer(self, context=None): + yield self + + @decorators.cachedproperty + def _proxied(self): # pylint: disable=method-hidden + ast_builtins = MANAGER.builtins_module + return ast_builtins.getattr("frozenset")[0] + + +class Super(node_classes.NodeNG): + """Proxy class over a super call. + + This class offers almost the same behaviour as Python's super, + which is MRO lookups for retrieving attributes from the parents. + + The *mro_pointer* is the place in the MRO from where we should + start looking, not counting it. *mro_type* is the object which + provides the MRO, it can be both a type or an instance. + *self_class* is the class where the super call is, while + *scope* is the function where the super call is. + """ + + # pylint: disable=unnecessary-lambda + special_attributes = util.lazy_descriptor(lambda: objectmodel.SuperModel()) + + # pylint: disable=super-init-not-called + def __init__(self, mro_pointer, mro_type, self_class, scope): + self.type = mro_type + self.mro_pointer = mro_pointer + self._class_based = False + self._self_class = self_class + self._scope = scope + + def _infer(self, context=None): + yield self + + def super_mro(self): + """Get the MRO which will be used to lookup attributes in this super.""" + if not isinstance(self.mro_pointer, scoped_nodes.ClassDef): + raise exceptions.SuperError( + "The first argument to super must be a subtype of " + "type, not {mro_pointer}.", + super_=self, + ) + + if isinstance(self.type, scoped_nodes.ClassDef): + # `super(type, type)`, most likely in a class method. + self._class_based = True + mro_type = self.type + else: + mro_type = getattr(self.type, "_proxied", None) + if not isinstance(mro_type, (bases.Instance, scoped_nodes.ClassDef)): + raise exceptions.SuperError( + "The second argument to super must be an " + "instance or subtype of type, not {type}.", + super_=self, + ) + + if not mro_type.newstyle: + raise exceptions.SuperError( + "Unable to call super on old-style classes.", super_=self + ) + + mro = mro_type.mro() + if self.mro_pointer not in mro: + raise exceptions.SuperError( + "The second argument to super must be an " + "instance or subtype of type, not {type}.", + super_=self, + ) + + index = mro.index(self.mro_pointer) + return mro[index + 1 :] + + @decorators.cachedproperty + def _proxied(self): + ast_builtins = MANAGER.builtins_module + return ast_builtins.getattr("super")[0] + + def pytype(self): + return "%s.super" % BUILTINS + + def display_type(self): + return "Super of" + + @property + def name(self): + """Get the name of the MRO pointer.""" + return self.mro_pointer.name + + def qname(self): + return "super" + + def igetattr(self, name, context=None): + """Retrieve the inferred values of the given attribute name.""" + + if name in self.special_attributes: + yield self.special_attributes.lookup(name) + return + + try: + mro = self.super_mro() + # Don't let invalid MROs or invalid super calls + # leak out as is from this function. + except exceptions.SuperError as exc: + raise exceptions.AttributeInferenceError( + ( + "Lookup for {name} on {target!r} because super call {super!r} " + "is invalid." + ), + target=self, + attribute=name, + context=context, + super_=exc.super_, + ) from exc + except exceptions.MroError as exc: + raise exceptions.AttributeInferenceError( + ( + "Lookup for {name} on {target!r} failed because {cls!r} has an " + "invalid MRO." + ), + target=self, + attribute=name, + context=context, + mros=exc.mros, + cls=exc.cls, + ) from exc + found = False + for cls in mro: + if name not in cls.locals: + continue + + found = True + for inferred in bases._infer_stmts([cls[name]], context, frame=self): + if not isinstance(inferred, scoped_nodes.FunctionDef): + yield inferred + continue + + # We can obtain different descriptors from a super depending + # on what we are accessing and where the super call is. + if inferred.type == "classmethod": + yield bases.BoundMethod(inferred, cls) + elif self._scope.type == "classmethod" and inferred.type == "method": + yield inferred + elif self._class_based or inferred.type == "staticmethod": + yield inferred + elif isinstance(inferred, Property): + function = inferred.function + try: + yield from function.infer_call_result( + caller=self, context=context + ) + except exceptions.InferenceError: + yield util.Uninferable + elif bases._is_property(inferred): + # TODO: support other descriptors as well. + try: + yield from inferred.infer_call_result(self, context) + except exceptions.InferenceError: + yield util.Uninferable + else: + yield bases.BoundMethod(inferred, cls) + + if not found: + raise exceptions.AttributeInferenceError( + target=self, attribute=name, context=context + ) + + def getattr(self, name, context=None): + return list(self.igetattr(name, context=context)) + + +class ExceptionInstance(bases.Instance): + """Class for instances of exceptions + + It has special treatment for some of the exceptions's attributes, + which are transformed at runtime into certain concrete objects, such as + the case of .args. + """ + + @decorators.cachedproperty + def special_attributes(self): + qname = self.qname() + instance = objectmodel.BUILTIN_EXCEPTIONS.get( + qname, objectmodel.ExceptionInstanceModel + ) + return instance()(self) + + +class DictInstance(bases.Instance): + """Special kind of instances for dictionaries + + This instance knows the underlying object model of the dictionaries, which means + that methods such as .values or .items can be properly inferred. + """ + + # pylint: disable=unnecessary-lambda + special_attributes = util.lazy_descriptor(lambda: objectmodel.DictModel()) + + +# Custom objects tailored for dictionaries, which are used to +# disambiguate between the types of Python 2 dict's method returns +# and Python 3 (where they return set like objects). +class DictItems(bases.Proxy): + __str__ = node_classes.NodeNG.__str__ + __repr__ = node_classes.NodeNG.__repr__ + + +class DictKeys(bases.Proxy): + __str__ = node_classes.NodeNG.__str__ + __repr__ = node_classes.NodeNG.__repr__ + + +class DictValues(bases.Proxy): + __str__ = node_classes.NodeNG.__str__ + __repr__ = node_classes.NodeNG.__repr__ + + +class PartialFunction(scoped_nodes.FunctionDef): + """A class representing partial function obtained via functools.partial""" + + def __init__( + self, call, name=None, doc=None, lineno=None, col_offset=None, parent=None + ): + super().__init__(name, doc, lineno, col_offset, parent) + self.filled_positionals = len(call.positional_arguments[1:]) + self.filled_args = call.positional_arguments[1:] + self.filled_keywords = call.keyword_arguments + + def infer_call_result(self, caller=None, context=None): + if context: + current_passed_keywords = { + keyword for (keyword, _) in context.callcontext.keywords + } + for keyword, value in self.filled_keywords.items(): + if keyword not in current_passed_keywords: + context.callcontext.keywords.append((keyword, value)) + + call_context_args = context.callcontext.args or [] + context.callcontext.args = self.filled_args + call_context_args + + return super().infer_call_result(caller=caller, context=context) + + def qname(self): + return self.__class__.__name__ + + +# TODO: Hack to solve the circular import problem between node_classes and objects +# This is not needed in 2.0, which has a cleaner design overall +node_classes.Dict.__bases__ = (node_classes.NodeNG, DictInstance) + + +class Property(scoped_nodes.FunctionDef): + """Class representing a Python property""" + + def __init__( + self, function, name=None, doc=None, lineno=None, col_offset=None, parent=None + ): + self.function = function + super().__init__(name, doc, lineno, col_offset, parent) + + # pylint: disable=unnecessary-lambda + special_attributes = util.lazy_descriptor(lambda: objectmodel.PropertyModel()) + type = "property" + + def pytype(self): + return "%s.property" % BUILTINS + + def infer_call_result(self, caller=None, context=None): + raise exceptions.InferenceError("Properties are not callable") + + def infer(self, context=None, **kwargs): + return iter((self,)) diff --git a/WebCrawler/venv/Lib/site-packages/astroid/protocols.py b/WebCrawler/venv/Lib/site-packages/astroid/protocols.py new file mode 100644 index 0000000..2cdf554 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/astroid/protocols.py @@ -0,0 +1,780 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2009-2011, 2013-2014 LOGILAB S.A. (Paris, FRANCE) +# Copyright (c) 2014-2020 Claudiu Popa +# Copyright (c) 2014 Google, Inc. +# Copyright (c) 2014 Eevee (Alex Munroe) +# Copyright (c) 2015-2016 Ceridwen +# Copyright (c) 2015 Dmitry Pribysh +# Copyright (c) 2016 Derek Gustafson +# Copyright (c) 2017-2018 Ashley Whetter +# Copyright (c) 2017 Łukasz Rogalski +# Copyright (c) 2017 rr- +# Copyright (c) 2018 Nick Drozd +# Copyright (c) 2018 Ville Skyttä +# Copyright (c) 2018 Bryce Guinta +# Copyright (c) 2018 HoverHell +# Copyright (c) 2019 Hugo van Kemenade + +# Licensed under the LGPL: https://www.gnu.org/licenses/old-licenses/lgpl-2.1.en.html +# For details: https://github.com/PyCQA/astroid/blob/master/COPYING.LESSER + +"""this module contains a set of functions to handle python protocols for nodes +where it makes sense. +""" + +import collections +import operator as operator_mod + +import itertools + +from astroid import Store +from astroid import arguments +from astroid import bases +from astroid import context as contextmod +from astroid import exceptions +from astroid import decorators +from astroid import node_classes +from astroid import helpers +from astroid import nodes +from astroid import util + +raw_building = util.lazy_import("raw_building") +objects = util.lazy_import("objects") + + +def _reflected_name(name): + return "__r" + name[2:] + + +def _augmented_name(name): + return "__i" + name[2:] + + +_CONTEXTLIB_MGR = "contextlib.contextmanager" +BIN_OP_METHOD = { + "+": "__add__", + "-": "__sub__", + "/": "__truediv__", + "//": "__floordiv__", + "*": "__mul__", + "**": "__pow__", + "%": "__mod__", + "&": "__and__", + "|": "__or__", + "^": "__xor__", + "<<": "__lshift__", + ">>": "__rshift__", + "@": "__matmul__", +} + +REFLECTED_BIN_OP_METHOD = { + key: _reflected_name(value) for (key, value) in BIN_OP_METHOD.items() +} +AUGMENTED_OP_METHOD = { + key + "=": _augmented_name(value) for (key, value) in BIN_OP_METHOD.items() +} + +UNARY_OP_METHOD = { + "+": "__pos__", + "-": "__neg__", + "~": "__invert__", + "not": None, # XXX not '__nonzero__' +} +_UNARY_OPERATORS = { + "+": operator_mod.pos, + "-": operator_mod.neg, + "~": operator_mod.invert, + "not": operator_mod.not_, +} + + +def _infer_unary_op(obj, op): + func = _UNARY_OPERATORS[op] + value = func(obj) + return nodes.const_factory(value) + + +nodes.Tuple.infer_unary_op = lambda self, op: _infer_unary_op(tuple(self.elts), op) +nodes.List.infer_unary_op = lambda self, op: _infer_unary_op(self.elts, op) +nodes.Set.infer_unary_op = lambda self, op: _infer_unary_op(set(self.elts), op) +nodes.Const.infer_unary_op = lambda self, op: _infer_unary_op(self.value, op) +nodes.Dict.infer_unary_op = lambda self, op: _infer_unary_op(dict(self.items), op) + +# Binary operations + +BIN_OP_IMPL = { + "+": lambda a, b: a + b, + "-": lambda a, b: a - b, + "/": lambda a, b: a / b, + "//": lambda a, b: a // b, + "*": lambda a, b: a * b, + "**": lambda a, b: a ** b, + "%": lambda a, b: a % b, + "&": lambda a, b: a & b, + "|": lambda a, b: a | b, + "^": lambda a, b: a ^ b, + "<<": lambda a, b: a << b, + ">>": lambda a, b: a >> b, + "@": operator_mod.matmul, +} +for _KEY, _IMPL in list(BIN_OP_IMPL.items()): + BIN_OP_IMPL[_KEY + "="] = _IMPL + + +@decorators.yes_if_nothing_inferred +def const_infer_binary_op(self, opnode, operator, other, context, _): + not_implemented = nodes.Const(NotImplemented) + if isinstance(other, nodes.Const): + try: + impl = BIN_OP_IMPL[operator] + try: + yield nodes.const_factory(impl(self.value, other.value)) + except TypeError: + # ArithmeticError is not enough: float >> float is a TypeError + yield not_implemented + except Exception: # pylint: disable=broad-except + yield util.Uninferable + except TypeError: + yield not_implemented + elif isinstance(self.value, str) and operator == "%": + # TODO(cpopa): implement string interpolation later on. + yield util.Uninferable + else: + yield not_implemented + + +nodes.Const.infer_binary_op = const_infer_binary_op + + +def _multiply_seq_by_int(self, opnode, other, context): + node = self.__class__(parent=opnode) + filtered_elts = ( + helpers.safe_infer(elt, context) or util.Uninferable + for elt in self.elts + if elt is not util.Uninferable + ) + node.elts = list(filtered_elts) * other.value + return node + + +def _filter_uninferable_nodes(elts, context): + for elt in elts: + if elt is util.Uninferable: + yield nodes.Unknown() + else: + for inferred in elt.infer(context): + if inferred is not util.Uninferable: + yield inferred + else: + yield nodes.Unknown() + + +@decorators.yes_if_nothing_inferred +def tl_infer_binary_op(self, opnode, operator, other, context, method): + not_implemented = nodes.Const(NotImplemented) + if isinstance(other, self.__class__) and operator == "+": + node = self.__class__(parent=opnode) + node.elts = list( + itertools.chain( + _filter_uninferable_nodes(self.elts, context), + _filter_uninferable_nodes(other.elts, context), + ) + ) + yield node + elif isinstance(other, nodes.Const) and operator == "*": + if not isinstance(other.value, int): + yield not_implemented + return + yield _multiply_seq_by_int(self, opnode, other, context) + elif isinstance(other, bases.Instance) and operator == "*": + # Verify if the instance supports __index__. + as_index = helpers.class_instance_as_index(other) + if not as_index: + yield util.Uninferable + else: + yield _multiply_seq_by_int(self, opnode, as_index, context) + else: + yield not_implemented + + +nodes.Tuple.infer_binary_op = tl_infer_binary_op +nodes.List.infer_binary_op = tl_infer_binary_op + + +@decorators.yes_if_nothing_inferred +def instance_class_infer_binary_op(self, opnode, operator, other, context, method): + return method.infer_call_result(self, context) + + +bases.Instance.infer_binary_op = instance_class_infer_binary_op +nodes.ClassDef.infer_binary_op = instance_class_infer_binary_op + + +# assignment ################################################################## + +"""the assigned_stmts method is responsible to return the assigned statement +(e.g. not inferred) according to the assignment type. + +The `assign_path` argument is used to record the lhs path of the original node. +For instance if we want assigned statements for 'c' in 'a, (b,c)', assign_path +will be [1, 1] once arrived to the Assign node. + +The `context` argument is the current inference context which should be given +to any intermediary inference necessary. +""" + + +def _resolve_looppart(parts, assign_path, context): + """recursive function to resolve multiple assignments on loops""" + assign_path = assign_path[:] + index = assign_path.pop(0) + for part in parts: + if part is util.Uninferable: + continue + if not hasattr(part, "itered"): + continue + try: + itered = part.itered() + except TypeError: + continue + for stmt in itered: + index_node = nodes.Const(index) + try: + assigned = stmt.getitem(index_node, context) + except ( + AttributeError, + exceptions.AstroidTypeError, + exceptions.AstroidIndexError, + ): + continue + if not assign_path: + # we achieved to resolved the assignment path, + # don't infer the last part + yield assigned + elif assigned is util.Uninferable: + break + else: + # we are not yet on the last part of the path + # search on each possibly inferred value + try: + yield from _resolve_looppart( + assigned.infer(context), assign_path, context + ) + except exceptions.InferenceError: + break + + +@decorators.raise_if_nothing_inferred +def for_assigned_stmts(self, node=None, context=None, assign_path=None): + if isinstance(self, nodes.AsyncFor) or getattr(self, "is_async", False): + # Skip inferring of async code for now + return dict(node=self, unknown=node, assign_path=assign_path, context=context) + if assign_path is None: + for lst in self.iter.infer(context): + if isinstance(lst, (nodes.Tuple, nodes.List)): + yield from lst.elts + else: + yield from _resolve_looppart(self.iter.infer(context), assign_path, context) + return dict(node=self, unknown=node, assign_path=assign_path, context=context) + + +nodes.For.assigned_stmts = for_assigned_stmts +nodes.Comprehension.assigned_stmts = for_assigned_stmts + + +def sequence_assigned_stmts(self, node=None, context=None, assign_path=None): + if assign_path is None: + assign_path = [] + try: + index = self.elts.index(node) + except ValueError as exc: + raise exceptions.InferenceError( + "Tried to retrieve a node {node!r} which does not exist", + node=self, + assign_path=assign_path, + context=context, + ) from exc + + assign_path.insert(0, index) + return self.parent.assigned_stmts( + node=self, context=context, assign_path=assign_path + ) + + +nodes.Tuple.assigned_stmts = sequence_assigned_stmts +nodes.List.assigned_stmts = sequence_assigned_stmts + + +def assend_assigned_stmts(self, node=None, context=None, assign_path=None): + return self.parent.assigned_stmts(node=self, context=context) + + +nodes.AssignName.assigned_stmts = assend_assigned_stmts +nodes.AssignAttr.assigned_stmts = assend_assigned_stmts + + +def _arguments_infer_argname(self, name, context): + # arguments information may be missing, in which case we can't do anything + # more + if not (self.arguments or self.vararg or self.kwarg): + yield util.Uninferable + return + + functype = self.parent.type + # first argument of instance/class method + if ( + self.arguments + and getattr(self.arguments[0], "name", None) == name + and functype != "staticmethod" + ): + cls = self.parent.parent.scope() + is_metaclass = isinstance(cls, nodes.ClassDef) and cls.type == "metaclass" + # If this is a metaclass, then the first argument will always + # be the class, not an instance. + if context.boundnode and isinstance(context.boundnode, bases.Instance): + cls = context.boundnode._proxied + if is_metaclass or functype == "classmethod": + yield cls + return + if functype == "method": + yield cls.instantiate_class() + return + + if context and context.callcontext: + call_site = arguments.CallSite(context.callcontext, context.extra_context) + yield from call_site.infer_argument(self.parent, name, context) + return + + if name == self.vararg: + vararg = nodes.const_factory(()) + vararg.parent = self + if not self.arguments and self.parent.name == "__init__": + cls = self.parent.parent.scope() + vararg.elts = [cls.instantiate_class()] + yield vararg + return + if name == self.kwarg: + kwarg = nodes.const_factory({}) + kwarg.parent = self + yield kwarg + return + # if there is a default value, yield it. And then yield Uninferable to reflect + # we can't guess given argument value + try: + context = contextmod.copy_context(context) + yield from self.default_value(name).infer(context) + yield util.Uninferable + except exceptions.NoDefault: + yield util.Uninferable + + +def arguments_assigned_stmts(self, node=None, context=None, assign_path=None): + if context.callcontext: + # reset call context/name + callcontext = context.callcontext + context = contextmod.copy_context(context) + context.callcontext = None + args = arguments.CallSite(callcontext, context=context) + return args.infer_argument(self.parent, node.name, context) + return _arguments_infer_argname(self, node.name, context) + + +nodes.Arguments.assigned_stmts = arguments_assigned_stmts + + +@decorators.raise_if_nothing_inferred +def assign_assigned_stmts(self, node=None, context=None, assign_path=None): + if not assign_path: + yield self.value + return None + yield from _resolve_assignment_parts( + self.value.infer(context), assign_path, context + ) + + return dict(node=self, unknown=node, assign_path=assign_path, context=context) + + +def assign_annassigned_stmts(self, node=None, context=None, assign_path=None): + for inferred in assign_assigned_stmts(self, node, context, assign_path): + if inferred is None: + yield util.Uninferable + else: + yield inferred + + +nodes.Assign.assigned_stmts = assign_assigned_stmts +nodes.AnnAssign.assigned_stmts = assign_annassigned_stmts +nodes.AugAssign.assigned_stmts = assign_assigned_stmts + + +def _resolve_assignment_parts(parts, assign_path, context): + """recursive function to resolve multiple assignments""" + assign_path = assign_path[:] + index = assign_path.pop(0) + for part in parts: + assigned = None + if isinstance(part, nodes.Dict): + # A dictionary in an iterating context + try: + assigned, _ = part.items[index] + except IndexError: + return + + elif hasattr(part, "getitem"): + index_node = nodes.Const(index) + try: + assigned = part.getitem(index_node, context) + except (exceptions.AstroidTypeError, exceptions.AstroidIndexError): + return + + if not assigned: + return + + if not assign_path: + # we achieved to resolved the assignment path, don't infer the + # last part + yield assigned + elif assigned is util.Uninferable: + return + else: + # we are not yet on the last part of the path search on each + # possibly inferred value + try: + yield from _resolve_assignment_parts( + assigned.infer(context), assign_path, context + ) + except exceptions.InferenceError: + return + + +@decorators.raise_if_nothing_inferred +def excepthandler_assigned_stmts(self, node=None, context=None, assign_path=None): + for assigned in node_classes.unpack_infer(self.type): + if isinstance(assigned, nodes.ClassDef): + assigned = objects.ExceptionInstance(assigned) + + yield assigned + return dict(node=self, unknown=node, assign_path=assign_path, context=context) + + +nodes.ExceptHandler.assigned_stmts = excepthandler_assigned_stmts + + +def _infer_context_manager(self, mgr, context): + inferred = next(mgr.infer(context=context)) + if isinstance(inferred, bases.Generator): + # Check if it is decorated with contextlib.contextmanager. + func = inferred.parent + if not func.decorators: + raise exceptions.InferenceError( + "No decorators found on inferred generator %s", node=func + ) + + for decorator_node in func.decorators.nodes: + decorator = next(decorator_node.infer(context=context)) + if isinstance(decorator, nodes.FunctionDef): + if decorator.qname() == _CONTEXTLIB_MGR: + break + else: + # It doesn't interest us. + raise exceptions.InferenceError(node=func) + + # Get the first yield point. If it has multiple yields, + # then a RuntimeError will be raised. + + possible_yield_points = func.nodes_of_class(nodes.Yield) + # Ignore yields in nested functions + yield_point = next( + (node for node in possible_yield_points if node.scope() == func), None + ) + if yield_point: + if not yield_point.value: + const = nodes.Const(None) + const.parent = yield_point + const.lineno = yield_point.lineno + yield const + else: + yield from yield_point.value.infer(context=context) + elif isinstance(inferred, bases.Instance): + try: + enter = next(inferred.igetattr("__enter__", context=context)) + except (exceptions.InferenceError, exceptions.AttributeInferenceError): + raise exceptions.InferenceError(node=inferred) + if not isinstance(enter, bases.BoundMethod): + raise exceptions.InferenceError(node=enter) + yield from enter.infer_call_result(self, context) + else: + raise exceptions.InferenceError(node=mgr) + + +@decorators.raise_if_nothing_inferred +def with_assigned_stmts(self, node=None, context=None, assign_path=None): + """Infer names and other nodes from a *with* statement. + + This enables only inference for name binding in a *with* statement. + For instance, in the following code, inferring `func` will return + the `ContextManager` class, not whatever ``__enter__`` returns. + We are doing this intentionally, because we consider that the context + manager result is whatever __enter__ returns and what it is binded + using the ``as`` keyword. + + class ContextManager(object): + def __enter__(self): + return 42 + with ContextManager() as f: + pass + + # ContextManager().infer() will return ContextManager + # f.infer() will return 42. + + Arguments: + self: nodes.With + node: The target of the assignment, `as (a, b)` in `with foo as (a, b)`. + context: Inference context used for caching already inferred objects + assign_path: + A list of indices, where each index specifies what item to fetch from + the inference results. + """ + try: + mgr = next(mgr for (mgr, vars) in self.items if vars == node) + except StopIteration: + return None + if assign_path is None: + yield from _infer_context_manager(self, mgr, context) + else: + for result in _infer_context_manager(self, mgr, context): + # Walk the assign_path and get the item at the final index. + obj = result + for index in assign_path: + if not hasattr(obj, "elts"): + raise exceptions.InferenceError( + "Wrong type ({targets!r}) for {node!r} assignment", + node=self, + targets=node, + assign_path=assign_path, + context=context, + ) + try: + obj = obj.elts[index] + except IndexError as exc: + raise exceptions.InferenceError( + "Tried to infer a nonexistent target with index {index} " + "in {node!r}.", + node=self, + targets=node, + assign_path=assign_path, + context=context, + ) from exc + except TypeError as exc: + raise exceptions.InferenceError( + "Tried to unpack a non-iterable value " "in {node!r}.", + node=self, + targets=node, + assign_path=assign_path, + context=context, + ) from exc + yield obj + return dict(node=self, unknown=node, assign_path=assign_path, context=context) + + +nodes.With.assigned_stmts = with_assigned_stmts + + +@decorators.raise_if_nothing_inferred +def named_expr_assigned_stmts(self, node, context=None, assign_path=None): + """Infer names and other nodes from an assignment expression""" + if self.target == node: + yield from self.value.infer(context=context) + else: + raise exceptions.InferenceError( + "Cannot infer NamedExpr node {node!r}", + node=self, + assign_path=assign_path, + context=context, + ) + + +nodes.NamedExpr.assigned_stmts = named_expr_assigned_stmts + + +@decorators.yes_if_nothing_inferred +def starred_assigned_stmts(self, node=None, context=None, assign_path=None): + """ + Arguments: + self: nodes.Starred + node: a node related to the current underlying Node. + context: Inference context used for caching already inferred objects + assign_path: + A list of indices, where each index specifies what item to fetch from + the inference results. + """ + # pylint: disable=too-many-locals,too-many-branches,too-many-statements + def _determine_starred_iteration_lookups(starred, target, lookups): + # Determine the lookups for the rhs of the iteration + itered = target.itered() + for index, element in enumerate(itered): + if ( + isinstance(element, nodes.Starred) + and element.value.name == starred.value.name + ): + lookups.append((index, len(itered))) + break + if isinstance(element, nodes.Tuple): + lookups.append((index, len(element.itered()))) + _determine_starred_iteration_lookups(starred, element, lookups) + + stmt = self.statement() + if not isinstance(stmt, (nodes.Assign, nodes.For)): + raise exceptions.InferenceError( + "Statement {stmt!r} enclosing {node!r} " "must be an Assign or For node.", + node=self, + stmt=stmt, + unknown=node, + context=context, + ) + + if context is None: + context = contextmod.InferenceContext() + + if isinstance(stmt, nodes.Assign): + value = stmt.value + lhs = stmt.targets[0] + + if sum(1 for _ in lhs.nodes_of_class(nodes.Starred)) > 1: + raise exceptions.InferenceError( + "Too many starred arguments in the " " assignment targets {lhs!r}.", + node=self, + targets=lhs, + unknown=node, + context=context, + ) + + try: + rhs = next(value.infer(context)) + except exceptions.InferenceError: + yield util.Uninferable + return + if rhs is util.Uninferable or not hasattr(rhs, "itered"): + yield util.Uninferable + return + + try: + elts = collections.deque(rhs.itered()) + except TypeError: + yield util.Uninferable + return + + # Unpack iteratively the values from the rhs of the assignment, + # until the find the starred node. What will remain will + # be the list of values which the Starred node will represent + # This is done in two steps, from left to right to remove + # anything before the starred node and from right to left + # to remove anything after the starred node. + + for index, left_node in enumerate(lhs.elts): + if not isinstance(left_node, nodes.Starred): + if not elts: + break + elts.popleft() + continue + lhs_elts = collections.deque(reversed(lhs.elts[index:])) + for right_node in lhs_elts: + if not isinstance(right_node, nodes.Starred): + if not elts: + break + elts.pop() + continue + + # We're done unpacking. + elts = list(elts) + packed = nodes.List( + ctx=Store, parent=self, lineno=lhs.lineno, col_offset=lhs.col_offset + ) + packed.postinit(elts=elts) + yield packed + break + + if isinstance(stmt, nodes.For): + try: + inferred_iterable = next(stmt.iter.infer(context=context)) + except exceptions.InferenceError: + yield util.Uninferable + return + if inferred_iterable is util.Uninferable or not hasattr( + inferred_iterable, "itered" + ): + yield util.Uninferable + return + try: + itered = inferred_iterable.itered() + except TypeError: + yield util.Uninferable + return + + target = stmt.target + + if not isinstance(target, nodes.Tuple): + raise exceptions.InferenceError( + "Could not make sense of this, the target must be a tuple", + context=context, + ) + + lookups = [] + _determine_starred_iteration_lookups(self, target, lookups) + if not lookups: + raise exceptions.InferenceError( + "Could not make sense of this, needs at least a lookup", context=context + ) + + # Make the last lookup a slice, since that what we want for a Starred node + last_element_index, last_element_length = lookups[-1] + is_starred_last = last_element_index == (last_element_length - 1) + + lookup_slice = slice( + last_element_index, + None if is_starred_last else (last_element_length - last_element_index), + ) + lookups[-1] = lookup_slice + + for element in itered: + + # We probably want to infer the potential values *for each* element in an + # iterable, but we can't infer a list of all values, when only a list of + # step values are expected: + # + # for a, *b in [...]: + # b + # + # *b* should now point to just the elements at that particular iteration step, + # which astroid can't know about. + + found_element = None + for lookup in lookups: + if not hasattr(element, "itered"): + break + if not isinstance(lookup, slice): + # Grab just the index, not the whole length + lookup = lookup[0] + try: + itered_inner_element = element.itered() + element = itered_inner_element[lookup] + except IndexError: + break + except TypeError: + # Most likely the itered() call failed, cannot make sense of this + yield util.Uninferable + return + else: + found_element = element + + unpacked = nodes.List( + ctx=Store, parent=self, lineno=self.lineno, col_offset=self.col_offset + ) + unpacked.postinit(elts=found_element or []) + yield unpacked + return + + yield util.Uninferable + + +nodes.Starred.assigned_stmts = starred_assigned_stmts diff --git a/WebCrawler/venv/Lib/site-packages/astroid/raw_building.py b/WebCrawler/venv/Lib/site-packages/astroid/raw_building.py new file mode 100644 index 0000000..b261277 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/astroid/raw_building.py @@ -0,0 +1,483 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2006-2014 LOGILAB S.A. (Paris, FRANCE) +# Copyright (c) 2012 FELD Boris +# Copyright (c) 2014-2020 Claudiu Popa +# Copyright (c) 2014 Google, Inc. +# Copyright (c) 2015-2016 Ceridwen +# Copyright (c) 2015 Florian Bruhin +# Copyright (c) 2015 Ovidiu Sabou +# Copyright (c) 2016 Derek Gustafson +# Copyright (c) 2016 Jakub Wilk +# Copyright (c) 2018 Ville Skyttä +# Copyright (c) 2018 Nick Drozd +# Copyright (c) 2018 Bryce Guinta +# Copyright (c) 2020 Robin Jarry + +# Licensed under the LGPL: https://www.gnu.org/licenses/old-licenses/lgpl-2.1.en.html +# For details: https://github.com/PyCQA/astroid/blob/master/COPYING.LESSER + +"""this module contains a set of functions to create astroid trees from scratch +(build_* functions) or from living object (object_build_* functions) +""" + +import builtins +import inspect +import os +import sys +import types + +from astroid import bases +from astroid import manager +from astroid import node_classes +from astroid import nodes + + +MANAGER = manager.AstroidManager() +# the keys of CONST_CLS eg python builtin types + +_CONSTANTS = tuple(node_classes.CONST_CLS) +_BUILTINS = vars(builtins) +TYPE_NONE = type(None) +TYPE_NOTIMPLEMENTED = type(NotImplemented) +TYPE_ELLIPSIS = type(...) + + +def _io_discrepancy(member): + # _io module names itself `io`: http://bugs.python.org/issue18602 + member_self = getattr(member, "__self__", None) + return ( + member_self + and inspect.ismodule(member_self) + and member_self.__name__ == "_io" + and member.__module__ == "io" + ) + + +def _attach_local_node(parent, node, name): + node.name = name # needed by add_local_node + parent.add_local_node(node) + + +def _add_dunder_class(func, member): + """Add a __class__ member to the given func node, if we can determine it.""" + python_cls = member.__class__ + cls_name = getattr(python_cls, "__name__", None) + if not cls_name: + return + cls_bases = [ancestor.__name__ for ancestor in python_cls.__bases__] + ast_klass = build_class(cls_name, cls_bases, python_cls.__doc__) + func.instance_attrs["__class__"] = [ast_klass] + + +_marker = object() + + +def attach_dummy_node(node, name, runtime_object=_marker): + """create a dummy node and register it in the locals of the given + node with the specified name + """ + enode = nodes.EmptyNode() + enode.object = runtime_object + _attach_local_node(node, enode, name) + + +def _has_underlying_object(self): + return self.object is not None and self.object is not _marker + + +nodes.EmptyNode.has_underlying_object = _has_underlying_object + + +def attach_const_node(node, name, value): + """create a Const node and register it in the locals of the given + node with the specified name + """ + if name not in node.special_attributes: + _attach_local_node(node, nodes.const_factory(value), name) + + +def attach_import_node(node, modname, membername): + """create a ImportFrom node and register it in the locals of the given + node with the specified name + """ + from_node = nodes.ImportFrom(modname, [(membername, None)]) + _attach_local_node(node, from_node, membername) + + +def build_module(name, doc=None): + """create and initialize an astroid Module node""" + node = nodes.Module(name, doc, pure_python=False) + node.package = False + node.parent = None + return node + + +def build_class(name, basenames=(), doc=None): + """create and initialize an astroid ClassDef node""" + node = nodes.ClassDef(name, doc) + for base in basenames: + basenode = nodes.Name() + basenode.name = base + node.bases.append(basenode) + basenode.parent = node + return node + + +def build_function(name, args=None, posonlyargs=None, defaults=None, doc=None): + """create and initialize an astroid FunctionDef node""" + args, defaults, posonlyargs = args or [], defaults or [], posonlyargs or [] + # first argument is now a list of decorators + func = nodes.FunctionDef(name, doc) + func.args = argsnode = nodes.Arguments() + argsnode.args = [] + argsnode.posonlyargs = [] + for arg in args: + argsnode.args.append(nodes.Name()) + argsnode.args[-1].name = arg + argsnode.args[-1].parent = argsnode + for arg in posonlyargs: + argsnode.posonlyargs.append(nodes.Name()) + argsnode.posonlyargs[-1].name = arg + argsnode.posonlyargs[-1].parent = argsnode + argsnode.defaults = [] + for default in defaults: + argsnode.defaults.append(nodes.const_factory(default)) + argsnode.defaults[-1].parent = argsnode + argsnode.kwarg = None + argsnode.vararg = None + argsnode.parent = func + if args: + register_arguments(func) + return func + + +def build_from_import(fromname, names): + """create and initialize an astroid ImportFrom import statement""" + return nodes.ImportFrom(fromname, [(name, None) for name in names]) + + +def register_arguments(func, args=None): + """add given arguments to local + + args is a list that may contains nested lists + (i.e. def func(a, (b, c, d)): ...) + """ + if args is None: + args = func.args.args + if func.args.vararg: + func.set_local(func.args.vararg, func.args) + if func.args.kwarg: + func.set_local(func.args.kwarg, func.args) + for arg in args: + if isinstance(arg, nodes.Name): + func.set_local(arg.name, arg) + else: + register_arguments(func, arg.elts) + + +def object_build_class(node, member, localname): + """create astroid for a living class object""" + basenames = [base.__name__ for base in member.__bases__] + return _base_class_object_build(node, member, basenames, localname=localname) + + +def object_build_function(node, member, localname): + """create astroid for a living function object""" + signature = inspect.signature(member) + args = [] + defaults = [] + posonlyargs = [] + for param_name, param in signature.parameters.items(): + if param.kind == inspect.Parameter.POSITIONAL_ONLY: + posonlyargs.append(param_name) + elif param.kind == inspect.Parameter.POSITIONAL_OR_KEYWORD: + args.append(param_name) + elif param.kind == inspect.Parameter.VAR_POSITIONAL: + args.append(param_name) + elif param.kind == inspect.Parameter.VAR_KEYWORD: + args.append(param_name) + if param.default is not inspect._empty: + defaults.append(param.default) + func = build_function( + getattr(member, "__name__", None) or localname, + args, + posonlyargs, + defaults, + member.__doc__, + ) + node.add_local_node(func, localname) + + +def object_build_datadescriptor(node, member, name): + """create astroid for a living data descriptor object""" + return _base_class_object_build(node, member, [], name) + + +def object_build_methoddescriptor(node, member, localname): + """create astroid for a living method descriptor object""" + # FIXME get arguments ? + func = build_function( + getattr(member, "__name__", None) or localname, doc=member.__doc__ + ) + # set node's arguments to None to notice that we have no information, not + # and empty argument list + func.args.args = None + node.add_local_node(func, localname) + _add_dunder_class(func, member) + + +def _base_class_object_build(node, member, basenames, name=None, localname=None): + """create astroid for a living class object, with a given set of base names + (e.g. ancestors) + """ + klass = build_class( + name or getattr(member, "__name__", None) or localname, + basenames, + member.__doc__, + ) + klass._newstyle = isinstance(member, type) + node.add_local_node(klass, localname) + try: + # limit the instantiation trick since it's too dangerous + # (such as infinite test execution...) + # this at least resolves common case such as Exception.args, + # OSError.errno + if issubclass(member, Exception): + instdict = member().__dict__ + else: + raise TypeError + except TypeError: + pass + else: + for item_name, obj in instdict.items(): + valnode = nodes.EmptyNode() + valnode.object = obj + valnode.parent = klass + valnode.lineno = 1 + klass.instance_attrs[item_name] = [valnode] + return klass + + +def _build_from_function(node, name, member, module): + # verify this is not an imported function + try: + code = member.__code__ + except AttributeError: + # Some implementations don't provide the code object, + # such as Jython. + code = None + filename = getattr(code, "co_filename", None) + if filename is None: + assert isinstance(member, object) + object_build_methoddescriptor(node, member, name) + elif filename != getattr(module, "__file__", None): + attach_dummy_node(node, name, member) + else: + object_build_function(node, member, name) + + +def _safe_has_attribute(obj, member): + try: + return hasattr(obj, member) + except Exception: # pylint: disable=broad-except + return False + + +class InspectBuilder: + """class for building nodes from living object + + this is actually a really minimal representation, including only Module, + FunctionDef and ClassDef nodes and some others as guessed. + """ + + def __init__(self): + self._done = {} + self._module = None + + def inspect_build(self, module, modname=None, path=None): + """build astroid from a living module (i.e. using inspect) + this is used when there is no python source code available (either + because it's a built-in module or because the .py is not available) + """ + self._module = module + if modname is None: + modname = module.__name__ + try: + node = build_module(modname, module.__doc__) + except AttributeError: + # in jython, java modules have no __doc__ (see #109562) + node = build_module(modname) + node.file = node.path = os.path.abspath(path) if path else path + node.name = modname + MANAGER.cache_module(node) + node.package = hasattr(module, "__path__") + self._done = {} + self.object_build(node, module) + return node + + def object_build(self, node, obj): + """recursive method which create a partial ast from real objects + (only function, class, and method are handled) + """ + if obj in self._done: + return self._done[obj] + self._done[obj] = node + for name in dir(obj): + try: + member = getattr(obj, name) + except AttributeError: + # damned ExtensionClass.Base, I know you're there ! + attach_dummy_node(node, name) + continue + if inspect.ismethod(member): + member = member.__func__ + if inspect.isfunction(member): + _build_from_function(node, name, member, self._module) + elif inspect.isbuiltin(member): + if not _io_discrepancy(member) and self.imported_member( + node, member, name + ): + continue + object_build_methoddescriptor(node, member, name) + elif inspect.isclass(member): + if self.imported_member(node, member, name): + continue + if member in self._done: + class_node = self._done[member] + if class_node not in node.locals.get(name, ()): + node.add_local_node(class_node, name) + else: + class_node = object_build_class(node, member, name) + # recursion + self.object_build(class_node, member) + if name == "__class__" and class_node.parent is None: + class_node.parent = self._done[self._module] + elif inspect.ismethoddescriptor(member): + assert isinstance(member, object) + object_build_methoddescriptor(node, member, name) + elif inspect.isdatadescriptor(member): + assert isinstance(member, object) + object_build_datadescriptor(node, member, name) + elif isinstance(member, _CONSTANTS): + attach_const_node(node, name, member) + elif inspect.isroutine(member): + # This should be called for Jython, where some builtin + # methods aren't caught by isbuiltin branch. + _build_from_function(node, name, member, self._module) + elif _safe_has_attribute(member, "__all__"): + module = build_module(name) + _attach_local_node(node, module, name) + # recursion + self.object_build(module, member) + else: + # create an empty node so that the name is actually defined + attach_dummy_node(node, name, member) + return None + + def imported_member(self, node, member, name): + """verify this is not an imported class or handle it""" + # /!\ some classes like ExtensionClass doesn't have a __module__ + # attribute ! Also, this may trigger an exception on badly built module + # (see http://www.logilab.org/ticket/57299 for instance) + try: + modname = getattr(member, "__module__", None) + except TypeError: + modname = None + if modname is None: + if name in ("__new__", "__subclasshook__"): + # Python 2.5.1 (r251:54863, Sep 1 2010, 22:03:14) + # >>> print object.__new__.__module__ + # None + modname = builtins.__name__ + else: + attach_dummy_node(node, name, member) + return True + + real_name = {"gtk": "gtk_gtk", "_io": "io"}.get(modname, modname) + + if real_name != self._module.__name__: + # check if it sounds valid and then add an import node, else use a + # dummy node + try: + getattr(sys.modules[modname], name) + except (KeyError, AttributeError): + attach_dummy_node(node, name, member) + else: + attach_import_node(node, modname, name) + return True + return False + + +### astroid bootstrapping ###################################################### + +_CONST_PROXY = {} + +# TODO : find a nicer way to handle this situation; +def _set_proxied(const): + return _CONST_PROXY[const.value.__class__] + + +def _astroid_bootstrapping(): + """astroid bootstrapping the builtins module""" + # this boot strapping is necessary since we need the Const nodes to + # inspect_build builtins, and then we can proxy Const + builder = InspectBuilder() + astroid_builtin = builder.inspect_build(builtins) + + # pylint: disable=redefined-outer-name + for cls, node_cls in node_classes.CONST_CLS.items(): + if cls is TYPE_NONE: + proxy = build_class("NoneType") + proxy.parent = astroid_builtin + elif cls is TYPE_NOTIMPLEMENTED: + proxy = build_class("NotImplementedType") + proxy.parent = astroid_builtin + elif cls is TYPE_ELLIPSIS: + proxy = build_class("Ellipsis") + proxy.parent = astroid_builtin + else: + proxy = astroid_builtin.getattr(cls.__name__)[0] + if cls in (dict, list, set, tuple): + node_cls._proxied = proxy + else: + _CONST_PROXY[cls] = proxy + + # Set the builtin module as parent for some builtins. + nodes.Const._proxied = property(_set_proxied) + + _GeneratorType = nodes.ClassDef( + types.GeneratorType.__name__, types.GeneratorType.__doc__ + ) + _GeneratorType.parent = astroid_builtin + bases.Generator._proxied = _GeneratorType + builder.object_build(bases.Generator._proxied, types.GeneratorType) + + if hasattr(types, "AsyncGeneratorType"): + # pylint: disable=no-member; AsyncGeneratorType + _AsyncGeneratorType = nodes.ClassDef( + types.AsyncGeneratorType.__name__, types.AsyncGeneratorType.__doc__ + ) + _AsyncGeneratorType.parent = astroid_builtin + bases.AsyncGenerator._proxied = _AsyncGeneratorType + builder.object_build(bases.AsyncGenerator._proxied, types.AsyncGeneratorType) + builtin_types = ( + types.GetSetDescriptorType, + types.GeneratorType, + types.MemberDescriptorType, + TYPE_NONE, + TYPE_NOTIMPLEMENTED, + types.FunctionType, + types.MethodType, + types.BuiltinFunctionType, + types.ModuleType, + types.TracebackType, + ) + for _type in builtin_types: + if _type.__name__ not in astroid_builtin: + cls = nodes.ClassDef(_type.__name__, _type.__doc__) + cls.parent = astroid_builtin + builder.object_build(cls, _type) + astroid_builtin[_type.__name__] = cls + + +_astroid_bootstrapping() diff --git a/WebCrawler/venv/Lib/site-packages/astroid/rebuilder.py b/WebCrawler/venv/Lib/site-packages/astroid/rebuilder.py new file mode 100644 index 0000000..3fc1a83 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/astroid/rebuilder.py @@ -0,0 +1,1010 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2009-2014 LOGILAB S.A. (Paris, FRANCE) +# Copyright (c) 2013-2020 Claudiu Popa +# Copyright (c) 2013-2014 Google, Inc. +# Copyright (c) 2014 Alexander Presnyakov +# Copyright (c) 2014 Eevee (Alex Munroe) +# Copyright (c) 2015-2016 Ceridwen +# Copyright (c) 2016-2017 Derek Gustafson +# Copyright (c) 2016 Jared Garst +# Copyright (c) 2017 Hugo +# Copyright (c) 2017 Łukasz Rogalski +# Copyright (c) 2017 rr- +# Copyright (c) 2018-2019 Ville Skyttä +# Copyright (c) 2018 Tomas Gavenciak +# Copyright (c) 2018 Serhiy Storchaka +# Copyright (c) 2018 Nick Drozd +# Copyright (c) 2018 Bryce Guinta +# Copyright (c) 2019-2020 Ashley Whetter +# Copyright (c) 2019 Hugo van Kemenade +# Copyright (c) 2019 Zbigniew Jędrzejewski-Szmek + +# Licensed under the LGPL: https://www.gnu.org/licenses/old-licenses/lgpl-2.1.en.html +# For details: https://github.com/PyCQA/astroid/blob/master/COPYING.LESSER + +"""this module contains utilities for rebuilding an _ast tree in +order to get a single Astroid representation +""" + +import sys +from typing import Optional + +import astroid +from astroid._ast import parse_function_type_comment, get_parser_module, ParserModule +from astroid import nodes + + +CONST_NAME_TRANSFORMS = {"None": None, "True": True, "False": False} + +REDIRECT = { + "arguments": "Arguments", + "comprehension": "Comprehension", + "ListCompFor": "Comprehension", + "GenExprFor": "Comprehension", + "excepthandler": "ExceptHandler", + "keyword": "Keyword", +} +PY37 = sys.version_info >= (3, 7) +PY38 = sys.version_info >= (3, 8) + + +def _visit_or_none(node, attr, visitor, parent, visit="visit", **kws): + """If the given node has an attribute, visits the attribute, and + otherwise returns None. + + """ + value = getattr(node, attr, None) + if value: + return getattr(visitor, visit)(value, parent, **kws) + + return None + + +class TreeRebuilder: + """Rebuilds the _ast tree to become an Astroid tree""" + + def __init__(self, manager, parser_module: Optional[ParserModule] = None): + self._manager = manager + self._global_names = [] + self._import_from_nodes = [] + self._delayed_assattr = [] + self._visit_meths = {} + + if parser_module is None: + self._parser_module = get_parser_module() + else: + self._parser_module = parser_module + self._module = self._parser_module.module + + def _get_doc(self, node): + try: + if PY37 and hasattr(node, "docstring"): + doc = node.docstring + return node, doc + if node.body and isinstance(node.body[0], self._module.Expr): + + first_value = node.body[0].value + if isinstance(first_value, self._module.Str) or ( + PY38 + and isinstance(first_value, self._module.Constant) + and isinstance(first_value.value, str) + ): + doc = first_value.value if PY38 else first_value.s + node.body = node.body[1:] + return node, doc + except IndexError: + pass # ast built from scratch + return node, None + + def _get_context(self, node): + return self._parser_module.context_classes.get(type(node.ctx), astroid.Load) + + def visit_module(self, node, modname, modpath, package): + """visit a Module node by returning a fresh instance of it""" + node, doc = self._get_doc(node) + newnode = nodes.Module( + name=modname, + doc=doc, + file=modpath, + path=[modpath], + package=package, + parent=None, + ) + newnode.postinit([self.visit(child, newnode) for child in node.body]) + return newnode + + def visit(self, node, parent): + cls = node.__class__ + if cls in self._visit_meths: + visit_method = self._visit_meths[cls] + else: + cls_name = cls.__name__ + visit_name = "visit_" + REDIRECT.get(cls_name, cls_name).lower() + visit_method = getattr(self, visit_name) + self._visit_meths[cls] = visit_method + return visit_method(node, parent) + + def _save_assignment(self, node, name=None): + """save assignement situation since node.parent is not available yet""" + if self._global_names and node.name in self._global_names[-1]: + node.root().set_local(node.name, node) + else: + node.parent.set_local(node.name, node) + + def visit_arg(self, node, parent): + """visit an arg node by returning a fresh AssName instance""" + return self.visit_assignname(node, parent, node.arg) + + def visit_arguments(self, node, parent): + """visit an Arguments node by returning a fresh instance of it""" + vararg, kwarg = node.vararg, node.kwarg + newnode = nodes.Arguments( + vararg.arg if vararg else None, kwarg.arg if kwarg else None, parent + ) + args = [self.visit(child, newnode) for child in node.args] + defaults = [self.visit(child, newnode) for child in node.defaults] + varargannotation = None + kwargannotation = None + posonlyargs = [] + # change added in 82732 (7c5c678e4164), vararg and kwarg + # are instances of `_ast.arg`, not strings + if vararg: + if node.vararg.annotation: + varargannotation = self.visit(node.vararg.annotation, newnode) + vararg = vararg.arg + if kwarg: + if node.kwarg.annotation: + kwargannotation = self.visit(node.kwarg.annotation, newnode) + kwarg = kwarg.arg + kwonlyargs = [self.visit(child, newnode) for child in node.kwonlyargs] + kw_defaults = [ + self.visit(child, newnode) if child else None for child in node.kw_defaults + ] + annotations = [ + self.visit(arg.annotation, newnode) if arg.annotation else None + for arg in node.args + ] + kwonlyargs_annotations = [ + self.visit(arg.annotation, newnode) if arg.annotation else None + for arg in node.kwonlyargs + ] + + posonlyargs_annotations = [] + if PY38: + posonlyargs = [self.visit(child, newnode) for child in node.posonlyargs] + posonlyargs_annotations = [ + self.visit(arg.annotation, newnode) if arg.annotation else None + for arg in node.posonlyargs + ] + type_comment_args = [ + self.check_type_comment(child, parent=newnode) for child in node.args + ] + type_comment_kwonlyargs = [ + self.check_type_comment(child, parent=newnode) for child in node.kwonlyargs + ] + type_comment_posonlyargs = [] + if PY38: + type_comment_posonlyargs = [ + self.check_type_comment(child, parent=newnode) + for child in node.posonlyargs + ] + + newnode.postinit( + args=args, + defaults=defaults, + kwonlyargs=kwonlyargs, + posonlyargs=posonlyargs, + kw_defaults=kw_defaults, + annotations=annotations, + kwonlyargs_annotations=kwonlyargs_annotations, + posonlyargs_annotations=posonlyargs_annotations, + varargannotation=varargannotation, + kwargannotation=kwargannotation, + type_comment_args=type_comment_args, + type_comment_kwonlyargs=type_comment_kwonlyargs, + type_comment_posonlyargs=type_comment_posonlyargs, + ) + # save argument names in locals: + if vararg: + newnode.parent.set_local(vararg, newnode) + if kwarg: + newnode.parent.set_local(kwarg, newnode) + return newnode + + def visit_assert(self, node, parent): + """visit a Assert node by returning a fresh instance of it""" + newnode = nodes.Assert(node.lineno, node.col_offset, parent) + if node.msg: + msg = self.visit(node.msg, newnode) + else: + msg = None + newnode.postinit(self.visit(node.test, newnode), msg) + return newnode + + def check_type_comment(self, node, parent): + type_comment = getattr(node, "type_comment", None) + if not type_comment: + return None + + try: + type_comment_ast = self._parser_module.parse(type_comment) + except SyntaxError: + # Invalid type comment, just skip it. + return None + + type_object = self.visit(type_comment_ast.body[0], parent=parent) + if not isinstance(type_object, nodes.Expr): + return None + + return type_object.value + + def check_function_type_comment(self, node): + type_comment = getattr(node, "type_comment", None) + if not type_comment: + return None + + try: + type_comment_ast = parse_function_type_comment(type_comment) + except SyntaxError: + # Invalid type comment, just skip it. + return None + + returns = None + argtypes = [ + self.visit(elem, node) for elem in (type_comment_ast.argtypes or []) + ] + if type_comment_ast.returns: + returns = self.visit(type_comment_ast.returns, node) + + return returns, argtypes + + # Async structs added in Python 3.5 + def visit_asyncfunctiondef(self, node, parent): + return self._visit_functiondef(nodes.AsyncFunctionDef, node, parent) + + def visit_asyncfor(self, node, parent): + return self._visit_for(nodes.AsyncFor, node, parent) + + def visit_await(self, node, parent): + newnode = nodes.Await(node.lineno, node.col_offset, parent) + newnode.postinit(value=self.visit(node.value, newnode)) + return newnode + + def visit_asyncwith(self, node, parent): + return self._visit_with(nodes.AsyncWith, node, parent) + + def visit_assign(self, node, parent): + """visit a Assign node by returning a fresh instance of it""" + newnode = nodes.Assign(node.lineno, node.col_offset, parent) + type_annotation = self.check_type_comment(node, parent=newnode) + newnode.postinit( + targets=[self.visit(child, newnode) for child in node.targets], + value=self.visit(node.value, newnode), + type_annotation=type_annotation, + ) + return newnode + + def visit_annassign(self, node, parent): + """visit an AnnAssign node by returning a fresh instance of it""" + newnode = nodes.AnnAssign(node.lineno, node.col_offset, parent) + annotation = _visit_or_none(node, "annotation", self, newnode) + newnode.postinit( + target=self.visit(node.target, newnode), + annotation=annotation, + simple=node.simple, + value=_visit_or_none(node, "value", self, newnode), + ) + return newnode + + def visit_assignname(self, node, parent, node_name=None): + """visit a node and return a AssignName node""" + newnode = nodes.AssignName( + node_name, + getattr(node, "lineno", None), + getattr(node, "col_offset", None), + parent, + ) + self._save_assignment(newnode) + return newnode + + def visit_augassign(self, node, parent): + """visit a AugAssign node by returning a fresh instance of it""" + newnode = nodes.AugAssign( + self._parser_module.bin_op_classes[type(node.op)] + "=", + node.lineno, + node.col_offset, + parent, + ) + newnode.postinit( + self.visit(node.target, newnode), self.visit(node.value, newnode) + ) + return newnode + + def visit_repr(self, node, parent): + """visit a Backquote node by returning a fresh instance of it""" + newnode = nodes.Repr(node.lineno, node.col_offset, parent) + newnode.postinit(self.visit(node.value, newnode)) + return newnode + + def visit_binop(self, node, parent): + """visit a BinOp node by returning a fresh instance of it""" + newnode = nodes.BinOp( + self._parser_module.bin_op_classes[type(node.op)], + node.lineno, + node.col_offset, + parent, + ) + newnode.postinit( + self.visit(node.left, newnode), self.visit(node.right, newnode) + ) + return newnode + + def visit_boolop(self, node, parent): + """visit a BoolOp node by returning a fresh instance of it""" + newnode = nodes.BoolOp( + self._parser_module.bool_op_classes[type(node.op)], + node.lineno, + node.col_offset, + parent, + ) + newnode.postinit([self.visit(child, newnode) for child in node.values]) + return newnode + + def visit_break(self, node, parent): + """visit a Break node by returning a fresh instance of it""" + return nodes.Break( + getattr(node, "lineno", None), getattr(node, "col_offset", None), parent + ) + + def visit_call(self, node, parent): + """visit a CallFunc node by returning a fresh instance of it""" + newnode = nodes.Call(node.lineno, node.col_offset, parent) + starargs = _visit_or_none(node, "starargs", self, newnode) + kwargs = _visit_or_none(node, "kwargs", self, newnode) + args = [self.visit(child, newnode) for child in node.args] + + if node.keywords: + keywords = [self.visit(child, newnode) for child in node.keywords] + else: + keywords = None + if starargs: + new_starargs = nodes.Starred( + col_offset=starargs.col_offset, + lineno=starargs.lineno, + parent=starargs.parent, + ) + new_starargs.postinit(value=starargs) + args.append(new_starargs) + if kwargs: + new_kwargs = nodes.Keyword( + arg=None, + col_offset=kwargs.col_offset, + lineno=kwargs.lineno, + parent=kwargs.parent, + ) + new_kwargs.postinit(value=kwargs) + if keywords: + keywords.append(new_kwargs) + else: + keywords = [new_kwargs] + + newnode.postinit(self.visit(node.func, newnode), args, keywords) + return newnode + + def visit_classdef(self, node, parent, newstyle=True): + """visit a ClassDef node to become astroid""" + node, doc = self._get_doc(node) + newnode = nodes.ClassDef(node.name, doc, node.lineno, node.col_offset, parent) + metaclass = None + for keyword in node.keywords: + if keyword.arg == "metaclass": + metaclass = self.visit(keyword, newnode).value + break + if node.decorator_list: + decorators = self.visit_decorators(node, newnode) + else: + decorators = None + newnode.postinit( + [self.visit(child, newnode) for child in node.bases], + [self.visit(child, newnode) for child in node.body], + decorators, + newstyle, + metaclass, + [ + self.visit(kwd, newnode) + for kwd in node.keywords + if kwd.arg != "metaclass" + ], + ) + return newnode + + def visit_const(self, node, parent): + """visit a Const node by returning a fresh instance of it""" + return nodes.Const( + node.value, + getattr(node, "lineno", None), + getattr(node, "col_offset", None), + parent, + ) + + def visit_continue(self, node, parent): + """visit a Continue node by returning a fresh instance of it""" + return nodes.Continue( + getattr(node, "lineno", None), getattr(node, "col_offset", None), parent + ) + + def visit_compare(self, node, parent): + """visit a Compare node by returning a fresh instance of it""" + newnode = nodes.Compare(node.lineno, node.col_offset, parent) + newnode.postinit( + self.visit(node.left, newnode), + [ + ( + self._parser_module.cmp_op_classes[op.__class__], + self.visit(expr, newnode), + ) + for (op, expr) in zip(node.ops, node.comparators) + ], + ) + return newnode + + def visit_comprehension(self, node, parent): + """visit a Comprehension node by returning a fresh instance of it""" + newnode = nodes.Comprehension(parent) + newnode.postinit( + self.visit(node.target, newnode), + self.visit(node.iter, newnode), + [self.visit(child, newnode) for child in node.ifs], + getattr(node, "is_async", None), + ) + return newnode + + def visit_decorators(self, node, parent): + """visit a Decorators node by returning a fresh instance of it""" + # /!\ node is actually an _ast.FunctionDef node while + # parent is an astroid.nodes.FunctionDef node + if PY38: + # Set the line number of the first decorator for Python 3.8+. + lineno = node.decorator_list[0].lineno + else: + lineno = node.lineno + newnode = nodes.Decorators(lineno, node.col_offset, parent) + newnode.postinit([self.visit(child, newnode) for child in node.decorator_list]) + return newnode + + def visit_delete(self, node, parent): + """visit a Delete node by returning a fresh instance of it""" + newnode = nodes.Delete(node.lineno, node.col_offset, parent) + newnode.postinit([self.visit(child, newnode) for child in node.targets]) + return newnode + + def _visit_dict_items(self, node, parent, newnode): + for key, value in zip(node.keys, node.values): + rebuilt_value = self.visit(value, newnode) + if not key: + # Python 3.5 and extended unpacking + rebuilt_key = nodes.DictUnpack( + rebuilt_value.lineno, rebuilt_value.col_offset, parent + ) + else: + rebuilt_key = self.visit(key, newnode) + yield rebuilt_key, rebuilt_value + + def visit_dict(self, node, parent): + """visit a Dict node by returning a fresh instance of it""" + newnode = nodes.Dict(node.lineno, node.col_offset, parent) + items = list(self._visit_dict_items(node, parent, newnode)) + newnode.postinit(items) + return newnode + + def visit_dictcomp(self, node, parent): + """visit a DictComp node by returning a fresh instance of it""" + newnode = nodes.DictComp(node.lineno, node.col_offset, parent) + newnode.postinit( + self.visit(node.key, newnode), + self.visit(node.value, newnode), + [self.visit(child, newnode) for child in node.generators], + ) + return newnode + + def visit_expr(self, node, parent): + """visit a Expr node by returning a fresh instance of it""" + newnode = nodes.Expr(node.lineno, node.col_offset, parent) + newnode.postinit(self.visit(node.value, newnode)) + return newnode + + # Not used in Python 3.8+. + def visit_ellipsis(self, node, parent): + """visit an Ellipsis node by returning a fresh instance of it""" + return nodes.Ellipsis( + getattr(node, "lineno", None), getattr(node, "col_offset", None), parent + ) + + def visit_emptynode(self, node, parent): + """visit an EmptyNode node by returning a fresh instance of it""" + return nodes.EmptyNode( + getattr(node, "lineno", None), getattr(node, "col_offset", None), parent + ) + + def visit_excepthandler(self, node, parent): + """visit an ExceptHandler node by returning a fresh instance of it""" + newnode = nodes.ExceptHandler(node.lineno, node.col_offset, parent) + if node.name: + name = self.visit_assignname(node, newnode, node.name) + else: + name = None + newnode.postinit( + _visit_or_none(node, "type", self, newnode), + name, + [self.visit(child, newnode) for child in node.body], + ) + return newnode + + def visit_exec(self, node, parent): + """visit an Exec node by returning a fresh instance of it""" + newnode = nodes.Exec(node.lineno, node.col_offset, parent) + newnode.postinit( + self.visit(node.body, newnode), + _visit_or_none(node, "globals", self, newnode), + _visit_or_none(node, "locals", self, newnode), + ) + return newnode + + # Not used in Python 3.8+. + def visit_extslice(self, node, parent): + """visit an ExtSlice node by returning a fresh instance of it""" + newnode = nodes.ExtSlice(parent=parent) + newnode.postinit([self.visit(dim, newnode) for dim in node.dims]) + return newnode + + def _visit_for(self, cls, node, parent): + """visit a For node by returning a fresh instance of it""" + newnode = cls(node.lineno, node.col_offset, parent) + type_annotation = self.check_type_comment(node, parent=newnode) + newnode.postinit( + target=self.visit(node.target, newnode), + iter=self.visit(node.iter, newnode), + body=[self.visit(child, newnode) for child in node.body], + orelse=[self.visit(child, newnode) for child in node.orelse], + type_annotation=type_annotation, + ) + return newnode + + def visit_for(self, node, parent): + return self._visit_for(nodes.For, node, parent) + + def visit_importfrom(self, node, parent): + """visit an ImportFrom node by returning a fresh instance of it""" + names = [(alias.name, alias.asname) for alias in node.names] + newnode = nodes.ImportFrom( + node.module or "", + names, + node.level or None, + getattr(node, "lineno", None), + getattr(node, "col_offset", None), + parent, + ) + # store From names to add them to locals after building + self._import_from_nodes.append(newnode) + return newnode + + def _visit_functiondef(self, cls, node, parent): + """visit an FunctionDef node to become astroid""" + self._global_names.append({}) + node, doc = self._get_doc(node) + + lineno = node.lineno + if PY38 and node.decorator_list: + # Python 3.8 sets the line number of a decorated function + # to be the actual line number of the function, but the + # previous versions expected the decorator's line number instead. + # We reset the function's line number to that of the + # first decorator to maintain backward compatibility. + # It's not ideal but this discrepancy was baked into + # the framework for *years*. + lineno = node.decorator_list[0].lineno + + newnode = cls(node.name, doc, lineno, node.col_offset, parent) + if node.decorator_list: + decorators = self.visit_decorators(node, newnode) + else: + decorators = None + if node.returns: + returns = self.visit(node.returns, newnode) + else: + returns = None + + type_comment_args = type_comment_returns = None + type_comment_annotation = self.check_function_type_comment(node) + if type_comment_annotation: + type_comment_returns, type_comment_args = type_comment_annotation + newnode.postinit( + args=self.visit(node.args, newnode), + body=[self.visit(child, newnode) for child in node.body], + decorators=decorators, + returns=returns, + type_comment_returns=type_comment_returns, + type_comment_args=type_comment_args, + ) + self._global_names.pop() + return newnode + + def visit_functiondef(self, node, parent): + return self._visit_functiondef(nodes.FunctionDef, node, parent) + + def visit_generatorexp(self, node, parent): + """visit a GeneratorExp node by returning a fresh instance of it""" + newnode = nodes.GeneratorExp(node.lineno, node.col_offset, parent) + newnode.postinit( + self.visit(node.elt, newnode), + [self.visit(child, newnode) for child in node.generators], + ) + return newnode + + def visit_attribute(self, node, parent): + """visit an Attribute node by returning a fresh instance of it""" + context = self._get_context(node) + if context == astroid.Del: + # FIXME : maybe we should reintroduce and visit_delattr ? + # for instance, deactivating assign_ctx + newnode = nodes.DelAttr(node.attr, node.lineno, node.col_offset, parent) + elif context == astroid.Store: + newnode = nodes.AssignAttr(node.attr, node.lineno, node.col_offset, parent) + # Prohibit a local save if we are in an ExceptHandler. + if not isinstance(parent, astroid.ExceptHandler): + self._delayed_assattr.append(newnode) + else: + newnode = nodes.Attribute(node.attr, node.lineno, node.col_offset, parent) + newnode.postinit(self.visit(node.value, newnode)) + return newnode + + def visit_global(self, node, parent): + """visit a Global node to become astroid""" + newnode = nodes.Global( + node.names, + getattr(node, "lineno", None), + getattr(node, "col_offset", None), + parent, + ) + if self._global_names: # global at the module level, no effect + for name in node.names: + self._global_names[-1].setdefault(name, []).append(newnode) + return newnode + + def visit_if(self, node, parent): + """visit an If node by returning a fresh instance of it""" + newnode = nodes.If(node.lineno, node.col_offset, parent) + newnode.postinit( + self.visit(node.test, newnode), + [self.visit(child, newnode) for child in node.body], + [self.visit(child, newnode) for child in node.orelse], + ) + return newnode + + def visit_ifexp(self, node, parent): + """visit a IfExp node by returning a fresh instance of it""" + newnode = nodes.IfExp(node.lineno, node.col_offset, parent) + newnode.postinit( + self.visit(node.test, newnode), + self.visit(node.body, newnode), + self.visit(node.orelse, newnode), + ) + return newnode + + def visit_import(self, node, parent): + """visit a Import node by returning a fresh instance of it""" + names = [(alias.name, alias.asname) for alias in node.names] + newnode = nodes.Import( + names, + getattr(node, "lineno", None), + getattr(node, "col_offset", None), + parent, + ) + # save import names in parent's locals: + for (name, asname) in newnode.names: + name = asname or name + parent.set_local(name.split(".")[0], newnode) + return newnode + + def visit_joinedstr(self, node, parent): + newnode = nodes.JoinedStr(node.lineno, node.col_offset, parent) + newnode.postinit([self.visit(child, newnode) for child in node.values]) + return newnode + + def visit_formattedvalue(self, node, parent): + newnode = nodes.FormattedValue(node.lineno, node.col_offset, parent) + newnode.postinit( + self.visit(node.value, newnode), + node.conversion, + _visit_or_none(node, "format_spec", self, newnode), + ) + return newnode + + def visit_namedexpr(self, node, parent): + newnode = nodes.NamedExpr(node.lineno, node.col_offset, parent) + newnode.postinit( + self.visit(node.target, newnode), self.visit(node.value, newnode) + ) + return newnode + + # Not used in Python 3.8+. + def visit_index(self, node, parent): + """visit a Index node by returning a fresh instance of it""" + newnode = nodes.Index(parent=parent) + newnode.postinit(self.visit(node.value, newnode)) + return newnode + + def visit_keyword(self, node, parent): + """visit a Keyword node by returning a fresh instance of it""" + newnode = nodes.Keyword(node.arg, parent=parent) + newnode.postinit(self.visit(node.value, newnode)) + return newnode + + def visit_lambda(self, node, parent): + """visit a Lambda node by returning a fresh instance of it""" + newnode = nodes.Lambda(node.lineno, node.col_offset, parent) + newnode.postinit(self.visit(node.args, newnode), self.visit(node.body, newnode)) + return newnode + + def visit_list(self, node, parent): + """visit a List node by returning a fresh instance of it""" + context = self._get_context(node) + newnode = nodes.List( + ctx=context, lineno=node.lineno, col_offset=node.col_offset, parent=parent + ) + newnode.postinit([self.visit(child, newnode) for child in node.elts]) + return newnode + + def visit_listcomp(self, node, parent): + """visit a ListComp node by returning a fresh instance of it""" + newnode = nodes.ListComp(node.lineno, node.col_offset, parent) + newnode.postinit( + self.visit(node.elt, newnode), + [self.visit(child, newnode) for child in node.generators], + ) + return newnode + + def visit_name(self, node, parent): + """visit a Name node by returning a fresh instance of it""" + context = self._get_context(node) + # True and False can be assigned to something in py2x, so we have to + # check first the context. + if context == astroid.Del: + newnode = nodes.DelName(node.id, node.lineno, node.col_offset, parent) + elif context == astroid.Store: + newnode = nodes.AssignName(node.id, node.lineno, node.col_offset, parent) + elif node.id in CONST_NAME_TRANSFORMS: + newnode = nodes.Const( + CONST_NAME_TRANSFORMS[node.id], + getattr(node, "lineno", None), + getattr(node, "col_offset", None), + parent, + ) + return newnode + else: + newnode = nodes.Name(node.id, node.lineno, node.col_offset, parent) + # XXX REMOVE me : + if context in (astroid.Del, astroid.Store): # 'Aug' ?? + self._save_assignment(newnode) + return newnode + + # Not used in Python 3.8+. + def visit_nameconstant(self, node, parent): + # in Python 3.4 we have NameConstant for True / False / None + return nodes.Const( + node.value, + getattr(node, "lineno", None), + getattr(node, "col_offset", None), + parent, + ) + + def visit_nonlocal(self, node, parent): + """visit a Nonlocal node and return a new instance of it""" + return nodes.Nonlocal( + node.names, + getattr(node, "lineno", None), + getattr(node, "col_offset", None), + parent, + ) + + def visit_constant(self, node, parent): + """visit a Constant node by returning a fresh instance of Const""" + return nodes.Const( + node.value, + getattr(node, "lineno", None), + getattr(node, "col_offset", None), + parent, + ) + + # Not used in Python 3.8+. + def visit_str(self, node, parent): + """visit a String/Bytes node by returning a fresh instance of Const""" + return nodes.Const( + node.s, + getattr(node, "lineno", None), + getattr(node, "col_offset", None), + parent, + ) + + visit_bytes = visit_str + + # Not used in Python 3.8+. + def visit_num(self, node, parent): + """visit a Num node by returning a fresh instance of Const""" + return nodes.Const( + node.n, + getattr(node, "lineno", None), + getattr(node, "col_offset", None), + parent, + ) + + def visit_pass(self, node, parent): + """visit a Pass node by returning a fresh instance of it""" + return nodes.Pass(node.lineno, node.col_offset, parent) + + def visit_print(self, node, parent): + """visit a Print node by returning a fresh instance of it""" + newnode = nodes.Print(node.nl, node.lineno, node.col_offset, parent) + newnode.postinit( + _visit_or_none(node, "dest", self, newnode), + [self.visit(child, newnode) for child in node.values], + ) + return newnode + + def visit_raise(self, node, parent): + """visit a Raise node by returning a fresh instance of it""" + newnode = nodes.Raise(node.lineno, node.col_offset, parent) + # no traceback; anyway it is not used in Pylint + newnode.postinit( + _visit_or_none(node, "exc", self, newnode), + _visit_or_none(node, "cause", self, newnode), + ) + return newnode + + def visit_return(self, node, parent): + """visit a Return node by returning a fresh instance of it""" + newnode = nodes.Return(node.lineno, node.col_offset, parent) + if node.value is not None: + newnode.postinit(self.visit(node.value, newnode)) + return newnode + + def visit_set(self, node, parent): + """visit a Set node by returning a fresh instance of it""" + newnode = nodes.Set(node.lineno, node.col_offset, parent) + newnode.postinit([self.visit(child, newnode) for child in node.elts]) + return newnode + + def visit_setcomp(self, node, parent): + """visit a SetComp node by returning a fresh instance of it""" + newnode = nodes.SetComp(node.lineno, node.col_offset, parent) + newnode.postinit( + self.visit(node.elt, newnode), + [self.visit(child, newnode) for child in node.generators], + ) + return newnode + + def visit_slice(self, node, parent): + """visit a Slice node by returning a fresh instance of it""" + newnode = nodes.Slice(parent=parent) + newnode.postinit( + _visit_or_none(node, "lower", self, newnode), + _visit_or_none(node, "upper", self, newnode), + _visit_or_none(node, "step", self, newnode), + ) + return newnode + + def visit_subscript(self, node, parent): + """visit a Subscript node by returning a fresh instance of it""" + context = self._get_context(node) + newnode = nodes.Subscript( + ctx=context, lineno=node.lineno, col_offset=node.col_offset, parent=parent + ) + newnode.postinit( + self.visit(node.value, newnode), self.visit(node.slice, newnode) + ) + return newnode + + def visit_starred(self, node, parent): + """visit a Starred node and return a new instance of it""" + context = self._get_context(node) + newnode = nodes.Starred( + ctx=context, lineno=node.lineno, col_offset=node.col_offset, parent=parent + ) + newnode.postinit(self.visit(node.value, newnode)) + return newnode + + def visit_tryexcept(self, node, parent): + """visit a TryExcept node by returning a fresh instance of it""" + newnode = nodes.TryExcept(node.lineno, node.col_offset, parent) + newnode.postinit( + [self.visit(child, newnode) for child in node.body], + [self.visit(child, newnode) for child in node.handlers], + [self.visit(child, newnode) for child in node.orelse], + ) + return newnode + + def visit_try(self, node, parent): + # python 3.3 introduce a new Try node replacing + # TryFinally/TryExcept nodes + if node.finalbody: + newnode = nodes.TryFinally(node.lineno, node.col_offset, parent) + if node.handlers: + body = [self.visit_tryexcept(node, newnode)] + else: + body = [self.visit(child, newnode) for child in node.body] + newnode.postinit(body, [self.visit(n, newnode) for n in node.finalbody]) + return newnode + if node.handlers: + return self.visit_tryexcept(node, parent) + return None + + def visit_tryfinally(self, node, parent): + """visit a TryFinally node by returning a fresh instance of it""" + newnode = nodes.TryFinally(node.lineno, node.col_offset, parent) + newnode.postinit( + [self.visit(child, newnode) for child in node.body], + [self.visit(n, newnode) for n in node.finalbody], + ) + return newnode + + def visit_tuple(self, node, parent): + """visit a Tuple node by returning a fresh instance of it""" + context = self._get_context(node) + newnode = nodes.Tuple( + ctx=context, lineno=node.lineno, col_offset=node.col_offset, parent=parent + ) + newnode.postinit([self.visit(child, newnode) for child in node.elts]) + return newnode + + def visit_unaryop(self, node, parent): + """visit a UnaryOp node by returning a fresh instance of it""" + newnode = nodes.UnaryOp( + self._parser_module.unary_op_classes[node.op.__class__], + node.lineno, + node.col_offset, + parent, + ) + newnode.postinit(self.visit(node.operand, newnode)) + return newnode + + def visit_while(self, node, parent): + """visit a While node by returning a fresh instance of it""" + newnode = nodes.While(node.lineno, node.col_offset, parent) + newnode.postinit( + self.visit(node.test, newnode), + [self.visit(child, newnode) for child in node.body], + [self.visit(child, newnode) for child in node.orelse], + ) + return newnode + + def _visit_with(self, cls, node, parent): + newnode = cls(node.lineno, node.col_offset, parent) + + def visit_child(child): + expr = self.visit(child.context_expr, newnode) + var = _visit_or_none(child, "optional_vars", self, newnode) + return expr, var + + type_annotation = self.check_type_comment(node, parent=newnode) + newnode.postinit( + items=[visit_child(child) for child in node.items], + body=[self.visit(child, newnode) for child in node.body], + type_annotation=type_annotation, + ) + return newnode + + def visit_with(self, node, parent): + return self._visit_with(nodes.With, node, parent) + + def visit_yield(self, node, parent): + """visit a Yield node by returning a fresh instance of it""" + newnode = nodes.Yield(node.lineno, node.col_offset, parent) + if node.value is not None: + newnode.postinit(self.visit(node.value, newnode)) + return newnode + + def visit_yieldfrom(self, node, parent): + newnode = nodes.YieldFrom(node.lineno, node.col_offset, parent) + if node.value is not None: + newnode.postinit(self.visit(node.value, newnode)) + return newnode diff --git a/WebCrawler/venv/Lib/site-packages/astroid/scoped_nodes.py b/WebCrawler/venv/Lib/site-packages/astroid/scoped_nodes.py new file mode 100644 index 0000000..8561e74 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/astroid/scoped_nodes.py @@ -0,0 +1,2927 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2006-2014 LOGILAB S.A. (Paris, FRANCE) +# Copyright (c) 2010 Daniel Harding +# Copyright (c) 2011, 2013-2015 Google, Inc. +# Copyright (c) 2013-2020 Claudiu Popa +# Copyright (c) 2013 Phil Schaf +# Copyright (c) 2014 Eevee (Alex Munroe) +# Copyright (c) 2015-2016 Florian Bruhin +# Copyright (c) 2015-2016 Ceridwen +# Copyright (c) 2015 Rene Zhang +# Copyright (c) 2015 Philip Lorenz +# Copyright (c) 2016-2017 Derek Gustafson +# Copyright (c) 2017-2018 Bryce Guinta +# Copyright (c) 2017-2018 Ashley Whetter +# Copyright (c) 2017 Łukasz Rogalski +# Copyright (c) 2017 David Euresti +# Copyright (c) 2018-2019 Nick Drozd +# Copyright (c) 2018 Ville Skyttä +# Copyright (c) 2018 Anthony Sottile +# Copyright (c) 2018 HoverHell +# Copyright (c) 2019 Hugo van Kemenade +# Copyright (c) 2019 Peter de Blanc + +# Licensed under the LGPL: https://www.gnu.org/licenses/old-licenses/lgpl-2.1.en.html +# For details: https://github.com/PyCQA/astroid/blob/master/COPYING.LESSER + + +""" +This module contains the classes for "scoped" node, i.e. which are opening a +new local scope in the language definition : Module, ClassDef, FunctionDef (and +Lambda, GeneratorExp, DictComp and SetComp to some extent). +""" + +import builtins +import sys +import io +import itertools +from typing import Optional, List + +from astroid import bases +from astroid import context as contextmod +from astroid import exceptions +from astroid import decorators as decorators_mod +from astroid.interpreter import objectmodel +from astroid.interpreter import dunder_lookup +from astroid import manager +from astroid import mixins +from astroid import node_classes +from astroid import util + + +BUILTINS = builtins.__name__ +ITER_METHODS = ("__iter__", "__getitem__") +EXCEPTION_BASE_CLASSES = frozenset({"Exception", "BaseException"}) +objects = util.lazy_import("objects") +BUILTIN_DESCRIPTORS = frozenset( + {"classmethod", "staticmethod", "builtins.classmethod", "builtins.staticmethod"} +) + + +def _c3_merge(sequences, cls, context): + """Merges MROs in *sequences* to a single MRO using the C3 algorithm. + + Adapted from http://www.python.org/download/releases/2.3/mro/. + + """ + result = [] + while True: + sequences = [s for s in sequences if s] # purge empty sequences + if not sequences: + return result + for s1 in sequences: # find merge candidates among seq heads + candidate = s1[0] + for s2 in sequences: + if candidate in s2[1:]: + candidate = None + break # reject the current head, it appears later + else: + break + if not candidate: + # Show all the remaining bases, which were considered as + # candidates for the next mro sequence. + raise exceptions.InconsistentMroError( + message="Cannot create a consistent method resolution order " + "for MROs {mros} of class {cls!r}.", + mros=sequences, + cls=cls, + context=context, + ) + + result.append(candidate) + # remove the chosen candidate + for seq in sequences: + if seq[0] == candidate: + del seq[0] + return None + + +def clean_duplicates_mro(sequences, cls, context): + for sequence in sequences: + names = [ + (node.lineno, node.qname()) if node.name else None for node in sequence + ] + last_index = dict(map(reversed, enumerate(names))) + if names and names[0] is not None and last_index[names[0]] != 0: + raise exceptions.DuplicateBasesError( + message="Duplicates found in MROs {mros} for {cls!r}.", + mros=sequences, + cls=cls, + context=context, + ) + yield [ + node + for i, (node, name) in enumerate(zip(sequence, names)) + if name is None or last_index[name] == i + ] + + +def function_to_method(n, klass): + if isinstance(n, FunctionDef): + if n.type == "classmethod": + return bases.BoundMethod(n, klass) + if n.type == "property": + return n + if n.type != "staticmethod": + return bases.UnboundMethod(n) + return n + + +MANAGER = manager.AstroidManager() + + +def builtin_lookup(name): + """lookup a name into the builtin module + return the list of matching statements and the astroid for the builtin + module + """ + builtin_astroid = MANAGER.ast_from_module(builtins) + if name == "__dict__": + return builtin_astroid, () + try: + stmts = builtin_astroid.locals[name] + except KeyError: + stmts = () + return builtin_astroid, stmts + + +# TODO move this Mixin to mixins.py; problem: 'FunctionDef' in _scope_lookup +class LocalsDictNodeNG(node_classes.LookupMixIn, node_classes.NodeNG): + """ this class provides locals handling common to Module, FunctionDef + and ClassDef nodes, including a dict like interface for direct access + to locals information + """ + + # attributes below are set by the builder module or by raw factories + + locals = {} + """A map of the name of a local variable to the node defining the local. + + :type: dict(str, NodeNG) + """ + + def qname(self): + """Get the 'qualified' name of the node. + + For example: module.name, module.class.name ... + + :returns: The qualified name. + :rtype: str + """ + # pylint: disable=no-member; github.com/pycqa/astroid/issues/278 + if self.parent is None: + return self.name + return "%s.%s" % (self.parent.frame().qname(), self.name) + + def frame(self): + """The first parent frame node. + + A frame node is a :class:`Module`, :class:`FunctionDef`, + or :class:`ClassDef`. + + :returns: The first parent frame node. + :rtype: Module or FunctionDef or ClassDef + """ + return self + + def scope(self): + """The first parent node defining a new scope. + + :returns: The first parent scope node. + :rtype: Module or FunctionDef or ClassDef or Lambda or GenExpr + """ + return self + + def _scope_lookup(self, node, name, offset=0): + """XXX method for interfacing the scope lookup""" + try: + stmts = node._filter_stmts(self.locals[name], self, offset) + except KeyError: + stmts = () + if stmts: + return self, stmts + if self.parent: # i.e. not Module + # nested scope: if parent scope is a function, that's fine + # else jump to the module + pscope = self.parent.scope() + if not pscope.is_function: + pscope = pscope.root() + return pscope.scope_lookup(node, name) + return builtin_lookup(name) # Module + + def set_local(self, name, stmt): + """Define that the given name is declared in the given statement node. + + .. seealso:: :meth:`scope` + + :param name: The name that is being defined. + :type name: str + + :param stmt: The statement that defines the given name. + :type stmt: NodeNG + """ + # assert not stmt in self.locals.get(name, ()), (self, stmt) + self.locals.setdefault(name, []).append(stmt) + + __setitem__ = set_local + + def _append_node(self, child): + """append a child, linking it in the tree""" + # pylint: disable=no-member; depending by the class + # which uses the current class as a mixin or base class. + # It's rewritten in 2.0, so it makes no sense for now + # to spend development time on it. + self.body.append(child) + child.parent = self + + def add_local_node(self, child_node, name=None): + """Append a child that should alter the locals of this scope node. + + :param child_node: The child node that will alter locals. + :type child_node: NodeNG + + :param name: The name of the local that will be altered by + the given child node. + :type name: str or None + """ + if name != "__class__": + # add __class__ node as a child will cause infinite recursion later! + self._append_node(child_node) + self.set_local(name or child_node.name, child_node) + + def __getitem__(self, item): + """The first node the defines the given local. + + :param item: The name of the locally defined object. + :type item: str + + :raises KeyError: If the name is not defined. + """ + return self.locals[item][0] + + def __iter__(self): + """Iterate over the names of locals defined in this scoped node. + + :returns: The names of the defined locals. + :rtype: iterable(str) + """ + return iter(self.keys()) + + def keys(self): + """The names of locals defined in this scoped node. + + :returns: The names of the defined locals. + :rtype: list(str) + """ + return list(self.locals.keys()) + + def values(self): + """The nodes that define the locals in this scoped node. + + :returns: The nodes that define locals. + :rtype: list(NodeNG) + """ + return [self[key] for key in self.keys()] + + def items(self): + """Get the names of the locals and the node that defines the local. + + :returns: The names of locals and their associated node. + :rtype: list(tuple(str, NodeNG)) + """ + return list(zip(self.keys(), self.values())) + + def __contains__(self, name): + """Check if a local is defined in this scope. + + :param name: The name of the local to check for. + :type name: str + + :returns: True if this node has a local of the given name, + False otherwise. + :rtype: bool + """ + return name in self.locals + + +class Module(LocalsDictNodeNG): + """Class representing an :class:`ast.Module` node. + + >>> node = astroid.extract_node('import astroid') + >>> node + + >>> node.parent + + """ + + _astroid_fields = ("body",) + + fromlineno = 0 + """The first line that this node appears on in the source code. + + :type: int or None + """ + lineno = 0 + """The line that this node appears on in the source code. + + :type: int or None + """ + + # attributes below are set by the builder module or by raw factories + + file = None + """The path to the file that this ast has been extracted from. + + This will be ``None`` when the representation has been built from a + built-in module. + + :type: str or None + """ + file_bytes = None + """The string/bytes that this ast was built from. + + :type: str or bytes or None + """ + file_encoding = None + """The encoding of the source file. + + This is used to get unicode out of a source file. + Python 2 only. + + :type: str or None + """ + name = None + """The name of the module. + + :type: str or None + """ + pure_python = None + """Whether the ast was built from source. + + :type: bool or None + """ + package = None + """Whether the node represents a package or a module. + + :type: bool or None + """ + globals = None + """A map of the name of a global variable to the node defining the global. + + :type: dict(str, NodeNG) + """ + + # Future imports + future_imports = None + """The imports from ``__future__``. + + :type: set(str) or None + """ + special_attributes = objectmodel.ModuleModel() + """The names of special attributes that this module has. + + :type: objectmodel.ModuleModel + """ + + # names of module attributes available through the global scope + scope_attrs = {"__name__", "__doc__", "__file__", "__path__", "__package__"} + """The names of module attributes available through the global scope. + + :type: str(str) + """ + + _other_fields = ( + "name", + "doc", + "file", + "path", + "package", + "pure_python", + "future_imports", + ) + _other_other_fields = ("locals", "globals") + + def __init__( + self, + name, + doc, + file=None, + path: Optional[List[str]] = None, + package=None, + parent=None, + pure_python=True, + ): + """ + :param name: The name of the module. + :type name: str + + :param doc: The module docstring. + :type doc: str + + :param file: The path to the file that this ast has been extracted from. + :type file: str or None + + :param path: + :type path: Optional[List[str]] + + :param package: Whether the node represents a package or a module. + :type package: bool or None + + :param parent: The parent node in the syntax tree. + :type parent: NodeNG or None + + :param pure_python: Whether the ast was built from source. + :type pure_python: bool or None + """ + self.name = name + self.doc = doc + self.file = file + self.path = path + self.package = package + self.parent = parent + self.pure_python = pure_python + self.locals = self.globals = {} + """A map of the name of a local variable to the node defining the local. + + :type: dict(str, NodeNG) + """ + self.body = [] + """The contents of the module. + + :type: list(NodeNG) or None + """ + self.future_imports = set() + + # pylint: enable=redefined-builtin + + def postinit(self, body=None): + """Do some setup after initialisation. + + :param body: The contents of the module. + :type body: list(NodeNG) or None + """ + self.body = body + + def _get_stream(self): + if self.file_bytes is not None: + return io.BytesIO(self.file_bytes) + if self.file is not None: + stream = open(self.file, "rb") + return stream + return None + + def stream(self): + """Get a stream to the underlying file or bytes. + + :type: file or io.BytesIO or None + """ + return self._get_stream() + + def block_range(self, lineno): + """Get a range from where this node starts to where this node ends. + + :param lineno: Unused. + :type lineno: int + + :returns: The range of line numbers that this node belongs to. + :rtype: tuple(int, int) + """ + return self.fromlineno, self.tolineno + + def scope_lookup(self, node, name, offset=0): + """Lookup where the given variable is assigned. + + :param node: The node to look for assignments up to. + Any assignments after the given node are ignored. + :type node: NodeNG + + :param name: The name of the variable to find assignments for. + :type name: str + + :param offset: The line offset to filter statements up to. + :type offset: int + + :returns: This scope node and the list of assignments associated to the + given name according to the scope where it has been found (locals, + globals or builtin). + :rtype: tuple(str, list(NodeNG)) + """ + if name in self.scope_attrs and name not in self.locals: + try: + return self, self.getattr(name) + except exceptions.AttributeInferenceError: + return self, () + return self._scope_lookup(node, name, offset) + + def pytype(self): + """Get the name of the type that this node represents. + + :returns: The name of the type. + :rtype: str + """ + return "%s.module" % BUILTINS + + def display_type(self): + """A human readable type of this node. + + :returns: The type of this node. + :rtype: str + """ + return "Module" + + def getattr(self, name, context=None, ignore_locals=False): + if not name: + raise exceptions.AttributeInferenceError( + target=self, attribute=name, context=context + ) + + result = [] + name_in_locals = name in self.locals + + if name in self.special_attributes and not ignore_locals and not name_in_locals: + result = [self.special_attributes.lookup(name)] + elif not ignore_locals and name_in_locals: + result = self.locals[name] + elif self.package: + try: + result = [self.import_module(name, relative_only=True)] + except (exceptions.AstroidBuildingError, SyntaxError) as exc: + raise exceptions.AttributeInferenceError( + target=self, attribute=name, context=context + ) from exc + result = [n for n in result if not isinstance(n, node_classes.DelName)] + if result: + return result + raise exceptions.AttributeInferenceError( + target=self, attribute=name, context=context + ) + + def igetattr(self, name, context=None): + """Infer the possible values of the given variable. + + :param name: The name of the variable to infer. + :type name: str + + :returns: The inferred possible values. + :rtype: iterable(NodeNG) or None + """ + # set lookup name since this is necessary to infer on import nodes for + # instance + context = contextmod.copy_context(context) + context.lookupname = name + try: + return bases._infer_stmts(self.getattr(name, context), context, frame=self) + except exceptions.AttributeInferenceError as error: + raise exceptions.InferenceError( + error.message, target=self, attribute=name, context=context + ) from error + + def fully_defined(self): + """Check if this module has been build from a .py file. + + If so, the module contains a complete representation, + including the code. + + :returns: True if the module has been built from a .py file. + :rtype: bool + """ + return self.file is not None and self.file.endswith(".py") + + def statement(self): + """The first parent node, including self, marked as statement node. + + :returns: The first parent statement. + :rtype: NodeNG + """ + return self + + def previous_sibling(self): + """The previous sibling statement. + + :returns: The previous sibling statement node. + :rtype: NodeNG or None + """ + + def next_sibling(self): + """The next sibling statement node. + + :returns: The next sibling statement node. + :rtype: NodeNG or None + """ + + _absolute_import_activated = True + + def absolute_import_activated(self): + """Whether :pep:`328` absolute import behaviour has been enabled. + + :returns: True if :pep:`328` has been enabled, False otherwise. + :rtype: bool + """ + return self._absolute_import_activated + + def import_module(self, modname, relative_only=False, level=None): + """Get the ast for a given module as if imported from this module. + + :param modname: The name of the module to "import". + :type modname: str + + :param relative_only: Whether to only consider relative imports. + :type relative_only: bool + + :param level: The level of relative import. + :type level: int or None + + :returns: The imported module ast. + :rtype: NodeNG + """ + if relative_only and level is None: + level = 0 + absmodname = self.relative_to_absolute_name(modname, level) + + try: + return MANAGER.ast_from_module_name(absmodname) + except exceptions.AstroidBuildingError: + # we only want to import a sub module or package of this module, + # skip here + if relative_only: + raise + return MANAGER.ast_from_module_name(modname) + + def relative_to_absolute_name(self, modname, level): + """Get the absolute module name for a relative import. + + The relative import can be implicit or explicit. + + :param modname: The module name to convert. + :type modname: str + + :param level: The level of relative import. + :type level: int + + :returns: The absolute module name. + :rtype: str + + :raises TooManyLevelsError: When the relative import refers to a + module too far above this one. + """ + # XXX this returns non sens when called on an absolute import + # like 'pylint.checkers.astroid.utils' + # XXX doesn't return absolute name if self.name isn't absolute name + if self.absolute_import_activated() and level is None: + return modname + if level: + if self.package: + level = level - 1 + if level and self.name.count(".") < level: + raise exceptions.TooManyLevelsError(level=level, name=self.name) + + package_name = self.name.rsplit(".", level)[0] + elif self.package: + package_name = self.name + else: + package_name = self.name.rsplit(".", 1)[0] + + if package_name: + if not modname: + return package_name + return "%s.%s" % (package_name, modname) + return modname + + def wildcard_import_names(self): + """The list of imported names when this module is 'wildcard imported'. + + It doesn't include the '__builtins__' name which is added by the + current CPython implementation of wildcard imports. + + :returns: The list of imported names. + :rtype: list(str) + """ + # We separate the different steps of lookup in try/excepts + # to avoid catching too many Exceptions + default = [name for name in self.keys() if not name.startswith("_")] + try: + all_values = self["__all__"] + except KeyError: + return default + + try: + explicit = next(all_values.assigned_stmts()) + except exceptions.InferenceError: + return default + except AttributeError: + # not an assignment node + # XXX infer? + return default + + # Try our best to detect the exported name. + inferred = [] + try: + explicit = next(explicit.infer()) + except exceptions.InferenceError: + return default + if not isinstance(explicit, (node_classes.Tuple, node_classes.List)): + return default + + str_const = lambda node: ( + isinstance(node, node_classes.Const) and isinstance(node.value, str) + ) + for node in explicit.elts: + if str_const(node): + inferred.append(node.value) + else: + try: + inferred_node = next(node.infer()) + except exceptions.InferenceError: + continue + if str_const(inferred_node): + inferred.append(inferred_node.value) + return inferred + + def public_names(self): + """The list of the names that are publicly available in this module. + + :returns: The list of publc names. + :rtype: list(str) + """ + return [name for name in self.keys() if not name.startswith("_")] + + def bool_value(self, context=None): + """Determine the boolean value of this node. + + :returns: The boolean value of this node. + For a :class:`Module` this is always ``True``. + :rtype: bool + """ + return True + + def get_children(self): + yield from self.body + + +class ComprehensionScope(LocalsDictNodeNG): + """Scoping for different types of comprehensions.""" + + def frame(self): + """The first parent frame node. + + A frame node is a :class:`Module`, :class:`FunctionDef`, + or :class:`ClassDef`. + + :returns: The first parent frame node. + :rtype: Module or FunctionDef or ClassDef + """ + return self.parent.frame() + + scope_lookup = LocalsDictNodeNG._scope_lookup + + +class GeneratorExp(ComprehensionScope): + """Class representing an :class:`ast.GeneratorExp` node. + + >>> node = astroid.extract_node('(thing for thing in things if thing)') + >>> node + + """ + + _astroid_fields = ("elt", "generators") + _other_other_fields = ("locals",) + elt = None + """The element that forms the output of the expression. + + :type: NodeNG or None + """ + generators = None + """The generators that are looped through. + + :type: list(Comprehension) or None + """ + + def __init__(self, lineno=None, col_offset=None, parent=None): + """ + :param lineno: The line that this node appears on in the source code. + :type lineno: int or None + + :param col_offset: The column that this node appears on in the + source code. + :type col_offset: int or None + + :param parent: The parent node in the syntax tree. + :type parent: NodeNG or None + """ + self.locals = {} + """A map of the name of a local variable to the node defining the local. + + :type: dict(str, NodeNG) + """ + + super().__init__(lineno, col_offset, parent) + + def postinit(self, elt=None, generators=None): + """Do some setup after initialisation. + + :param elt: The element that forms the output of the expression. + :type elt: NodeNG or None + + :param generators: The generators that are looped through. + :type generators: list(Comprehension) or None + """ + self.elt = elt + if generators is None: + self.generators = [] + else: + self.generators = generators + + def bool_value(self, context=None): + """Determine the boolean value of this node. + + :returns: The boolean value of this node. + For a :class:`GeneratorExp` this is always ``True``. + :rtype: bool + """ + return True + + def get_children(self): + yield self.elt + + yield from self.generators + + +class DictComp(ComprehensionScope): + """Class representing an :class:`ast.DictComp` node. + + >>> node = astroid.extract_node('{k:v for k, v in things if k > v}') + >>> node + + """ + + _astroid_fields = ("key", "value", "generators") + _other_other_fields = ("locals",) + key = None + """What produces the keys. + + :type: NodeNG or None + """ + value = None + """What produces the values. + + :type: NodeNG or None + """ + generators = None + """The generators that are looped through. + + :type: list(Comprehension) or None + """ + + def __init__(self, lineno=None, col_offset=None, parent=None): + """ + :param lineno: The line that this node appears on in the source code. + :type lineno: int or None + + :param col_offset: The column that this node appears on in the + source code. + :type col_offset: int or None + + :param parent: The parent node in the syntax tree. + :type parent: NodeNG or None + """ + self.locals = {} + """A map of the name of a local variable to the node defining the local. + + :type: dict(str, NodeNG) + """ + + super().__init__(lineno, col_offset, parent) + + def postinit(self, key=None, value=None, generators=None): + """Do some setup after initialisation. + + :param key: What produces the keys. + :type key: NodeNG or None + + :param value: What produces the values. + :type value: NodeNG or None + + :param generators: The generators that are looped through. + :type generators: list(Comprehension) or None + """ + self.key = key + self.value = value + if generators is None: + self.generators = [] + else: + self.generators = generators + + def bool_value(self, context=None): + """Determine the boolean value of this node. + + :returns: The boolean value of this node. + For a :class:`DictComp` this is always :class:`Uninferable`. + :rtype: Uninferable + """ + return util.Uninferable + + def get_children(self): + yield self.key + yield self.value + + yield from self.generators + + +class SetComp(ComprehensionScope): + """Class representing an :class:`ast.SetComp` node. + + >>> node = astroid.extract_node('{thing for thing in things if thing}') + >>> node + + """ + + _astroid_fields = ("elt", "generators") + _other_other_fields = ("locals",) + elt = None + """The element that forms the output of the expression. + + :type: NodeNG or None + """ + generators = None + """The generators that are looped through. + + :type: list(Comprehension) or None + """ + + def __init__(self, lineno=None, col_offset=None, parent=None): + """ + :param lineno: The line that this node appears on in the source code. + :type lineno: int or None + + :param col_offset: The column that this node appears on in the + source code. + :type col_offset: int or None + + :param parent: The parent node in the syntax tree. + :type parent: NodeNG or None + """ + self.locals = {} + """A map of the name of a local variable to the node defining the local. + + :type: dict(str, NodeNG) + """ + + super().__init__(lineno, col_offset, parent) + + def postinit(self, elt=None, generators=None): + """Do some setup after initialisation. + + :param elt: The element that forms the output of the expression. + :type elt: NodeNG or None + + :param generators: The generators that are looped through. + :type generators: list(Comprehension) or None + """ + self.elt = elt + if generators is None: + self.generators = [] + else: + self.generators = generators + + def bool_value(self, context=None): + """Determine the boolean value of this node. + + :returns: The boolean value of this node. + For a :class:`SetComp` this is always :class:`Uninferable`. + :rtype: Uninferable + """ + return util.Uninferable + + def get_children(self): + yield self.elt + + yield from self.generators + + +class _ListComp(node_classes.NodeNG): + """Class representing an :class:`ast.ListComp` node. + + >>> node = astroid.extract_node('[thing for thing in things if thing]') + >>> node + + """ + + _astroid_fields = ("elt", "generators") + elt = None + """The element that forms the output of the expression. + + :type: NodeNG or None + """ + generators = None + """The generators that are looped through. + + :type: list(Comprehension) or None + """ + + def postinit(self, elt=None, generators=None): + """Do some setup after initialisation. + + :param elt: The element that forms the output of the expression. + :type elt: NodeNG or None + + :param generators: The generators that are looped through. + :type generators: list(Comprehension) or None + """ + self.elt = elt + self.generators = generators + + def bool_value(self, context=None): + """Determine the boolean value of this node. + + :returns: The boolean value of this node. + For a :class:`ListComp` this is always :class:`Uninferable`. + :rtype: Uninferable + """ + return util.Uninferable + + def get_children(self): + yield self.elt + + yield from self.generators + + +class ListComp(_ListComp, ComprehensionScope): + """Class representing an :class:`ast.ListComp` node. + + >>> node = astroid.extract_node('[thing for thing in things if thing]') + >>> node + + """ + + _other_other_fields = ("locals",) + + def __init__(self, lineno=None, col_offset=None, parent=None): + self.locals = {} + """A map of the name of a local variable to the node defining it. + + :type: dict(str, NodeNG) + """ + + super().__init__(lineno, col_offset, parent) + + +def _infer_decorator_callchain(node): + """Detect decorator call chaining and see if the end result is a + static or a classmethod. + """ + if not isinstance(node, FunctionDef): + return None + if not node.parent: + return None + try: + result = next(node.infer_call_result(node.parent)) + except exceptions.InferenceError: + return None + if isinstance(result, bases.Instance): + result = result._proxied + if isinstance(result, ClassDef): + if result.is_subtype_of("%s.classmethod" % BUILTINS): + return "classmethod" + if result.is_subtype_of("%s.staticmethod" % BUILTINS): + return "staticmethod" + if isinstance(result, FunctionDef): + if not result.decorators: + return None + # Determine if this function is decorated with one of the builtin descriptors we want. + for decorator in result.decorators.nodes: + if isinstance(decorator, node_classes.Name): + if decorator.name in BUILTIN_DESCRIPTORS: + return decorator.name + if ( + isinstance(decorator, node_classes.Attribute) + and isinstance(decorator.expr, node_classes.Name) + and decorator.expr.name == BUILTINS + and decorator.attrname in BUILTIN_DESCRIPTORS + ): + return decorator.attrname + return None + + +class Lambda(mixins.FilterStmtsMixin, LocalsDictNodeNG): + """Class representing an :class:`ast.Lambda` node. + + >>> node = astroid.extract_node('lambda arg: arg + 1') + >>> node + l.1 at 0x7f23b2e41518> + """ + + _astroid_fields = ("args", "body") + _other_other_fields = ("locals",) + name = "" + is_lambda = True + + def implicit_parameters(self): + return 0 + + # function's type, 'function' | 'method' | 'staticmethod' | 'classmethod' + @property + def type(self): + """Whether this is a method or function. + + :returns: 'method' if this is a method, 'function' otherwise. + :rtype: str + """ + # pylint: disable=no-member + if self.args.arguments and self.args.arguments[0].name == "self": + if isinstance(self.parent.scope(), ClassDef): + return "method" + return "function" + + def __init__(self, lineno=None, col_offset=None, parent=None): + """ + :param lineno: The line that this node appears on in the source code. + :type lineno: int or None + + :param col_offset: The column that this node appears on in the + source code. + :type col_offset: int or None + + :param parent: The parent node in the syntax tree. + :type parent: NodeNG or None + """ + self.locals = {} + """A map of the name of a local variable to the node defining it. + + :type: dict(str, NodeNG) + """ + + self.args = [] + """The arguments that the function takes. + + :type: Arguments or list + """ + + self.body = [] + """The contents of the function body. + + :type: list(NodeNG) + """ + + super().__init__(lineno, col_offset, parent) + + def postinit(self, args, body): + """Do some setup after initialisation. + + :param args: The arguments that the function takes. + :type args: Arguments + + :param body: The contents of the function body. + :type body: list(NodeNG) + """ + self.args = args + self.body = body + + def pytype(self): + """Get the name of the type that this node represents. + + :returns: The name of the type. + :rtype: str + """ + if "method" in self.type: + return "%s.instancemethod" % BUILTINS + return "%s.function" % BUILTINS + + def display_type(self): + """A human readable type of this node. + + :returns: The type of this node. + :rtype: str + """ + if "method" in self.type: + return "Method" + return "Function" + + def callable(self): + """Whether this node defines something that is callable. + + :returns: True if this defines something that is callable, + False otherwise. + For a :class:`Lambda` this is always ``True``. + :rtype: bool + """ + return True + + def argnames(self): + """Get the names of each of the arguments. + + :returns: The names of the arguments. + :rtype: list(str) + """ + # pylint: disable=no-member; github.com/pycqa/astroid/issues/291 + # args is in fact redefined later on by postinit. Can't be changed + # to None due to a strong interaction between Lambda and FunctionDef. + + if self.args.arguments: # maybe None with builtin functions + names = _rec_get_names(self.args.arguments) + else: + names = [] + if self.args.vararg: + names.append(self.args.vararg) + if self.args.kwarg: + names.append(self.args.kwarg) + return names + + def infer_call_result(self, caller, context=None): + """Infer what the function returns when called. + + :param caller: Unused + :type caller: object + """ + # pylint: disable=no-member; github.com/pycqa/astroid/issues/291 + # args is in fact redefined later on by postinit. Can't be changed + # to None due to a strong interaction between Lambda and FunctionDef. + return self.body.infer(context) + + def scope_lookup(self, node, name, offset=0): + """Lookup where the given names is assigned. + + :param node: The node to look for assignments up to. + Any assignments after the given node are ignored. + :type node: NodeNG + + :param name: The name to find assignments for. + :type name: str + + :param offset: The line offset to filter statements up to. + :type offset: int + + :returns: This scope node and the list of assignments associated to the + given name according to the scope where it has been found (locals, + globals or builtin). + :rtype: tuple(str, list(NodeNG)) + """ + # pylint: disable=no-member; github.com/pycqa/astroid/issues/291 + # args is in fact redefined later on by postinit. Can't be changed + # to None due to a strong interaction between Lambda and FunctionDef. + + if node in self.args.defaults or node in self.args.kw_defaults: + frame = self.parent.frame() + # line offset to avoid that def func(f=func) resolve the default + # value to the defined function + offset = -1 + else: + # check this is not used in function decorators + frame = self + return frame._scope_lookup(node, name, offset) + + def bool_value(self, context=None): + """Determine the boolean value of this node. + + :returns: The boolean value of this node. + For a :class:`Lambda` this is always ``True``. + :rtype: bool + """ + return True + + def get_children(self): + yield self.args + yield self.body + + +class FunctionDef(mixins.MultiLineBlockMixin, node_classes.Statement, Lambda): + """Class representing an :class:`ast.FunctionDef`. + + >>> node = astroid.extract_node(''' + ... def my_func(arg): + ... return arg + 1 + ... ''') + >>> node + + """ + + _astroid_fields = ("decorators", "args", "returns", "body") + _multi_line_block_fields = ("body",) + returns = None + decorators = None + """The decorators that are applied to this method or function. + + :type: Decorators or None + """ + special_attributes = objectmodel.FunctionModel() + """The names of special attributes that this function has. + + :type: objectmodel.FunctionModel + """ + is_function = True + """Whether this node indicates a function. + + For a :class:`FunctionDef` this is always ``True``. + + :type: bool + """ + type_annotation = None + """If present, this will contain the type annotation passed by a type comment + + :type: NodeNG or None + """ + type_comment_args = None + """ + If present, this will contain the type annotation for arguments + passed by a type comment + """ + type_comment_returns = None + """If present, this will contain the return type annotation, passed by a type comment""" + # attributes below are set by the builder module or by raw factories + _other_fields = ("name", "doc") + _other_other_fields = ( + "locals", + "_type", + "type_comment_returns", + "type_comment_args", + ) + _type = None + + def __init__(self, name=None, doc=None, lineno=None, col_offset=None, parent=None): + """ + :param name: The name of the function. + :type name: str or None + + :param doc: The function's docstring. + :type doc: str or None + + :param lineno: The line that this node appears on in the source code. + :type lineno: int or None + + :param col_offset: The column that this node appears on in the + source code. + :type col_offset: int or None + + :param parent: The parent node in the syntax tree. + :type parent: NodeNG or None + """ + self.name = name + """The name of the function. + + :type name: str or None + """ + + self.doc = doc + """The function's docstring. + + :type doc: str or None + """ + + self.instance_attrs = {} + super().__init__(lineno, col_offset, parent) + if parent: + frame = parent.frame() + frame.set_local(name, self) + + # pylint: disable=arguments-differ; different than Lambdas + def postinit( + self, + args, + body, + decorators=None, + returns=None, + type_comment_returns=None, + type_comment_args=None, + ): + """Do some setup after initialisation. + + :param args: The arguments that the function takes. + :type args: Arguments or list + + :param body: The contents of the function body. + :type body: list(NodeNG) + + :param decorators: The decorators that are applied to this + method or function. + :type decorators: Decorators or None + :params type_comment_returns: + The return type annotation passed via a type comment. + :params type_comment_args: + The args type annotation passed via a type comment. + """ + self.args = args + self.body = body + self.decorators = decorators + self.returns = returns + self.type_comment_returns = type_comment_returns + self.type_comment_args = type_comment_args + + @decorators_mod.cachedproperty + def extra_decorators(self): + """The extra decorators that this function can have. + + Additional decorators are considered when they are used as + assignments, as in ``method = staticmethod(method)``. + The property will return all the callables that are used for + decoration. + + :type: list(NodeNG) + """ + frame = self.parent.frame() + if not isinstance(frame, ClassDef): + return [] + + decorators = [] + for assign in frame._get_assign_nodes(): + if isinstance(assign.value, node_classes.Call) and isinstance( + assign.value.func, node_classes.Name + ): + for assign_node in assign.targets: + if not isinstance(assign_node, node_classes.AssignName): + # Support only `name = callable(name)` + continue + + if assign_node.name != self.name: + # Interested only in the assignment nodes that + # decorates the current method. + continue + try: + meth = frame[self.name] + except KeyError: + continue + else: + # Must be a function and in the same frame as the + # original method. + if ( + isinstance(meth, FunctionDef) + and assign_node.frame() == frame + ): + decorators.append(assign.value) + return decorators + + @decorators_mod.cachedproperty + def type( + self + ): # pylint: disable=invalid-overridden-method,too-many-return-statements + """The function type for this node. + + Possible values are: method, function, staticmethod, classmethod. + + :type: str + """ + for decorator in self.extra_decorators: + if decorator.func.name in BUILTIN_DESCRIPTORS: + return decorator.func.name + + frame = self.parent.frame() + type_name = "function" + if isinstance(frame, ClassDef): + if self.name == "__new__": + return "classmethod" + if sys.version_info >= (3, 6) and self.name == "__init_subclass__": + return "classmethod" + + type_name = "method" + + if not self.decorators: + return type_name + + for node in self.decorators.nodes: + if isinstance(node, node_classes.Name): + if node.name in BUILTIN_DESCRIPTORS: + return node.name + if ( + isinstance(node, node_classes.Attribute) + and isinstance(node.expr, node_classes.Name) + and node.expr.name == BUILTINS + and node.attrname in BUILTIN_DESCRIPTORS + ): + return node.attrname + + if isinstance(node, node_classes.Call): + # Handle the following case: + # @some_decorator(arg1, arg2) + # def func(...) + # + try: + current = next(node.func.infer()) + except exceptions.InferenceError: + continue + _type = _infer_decorator_callchain(current) + if _type is not None: + return _type + + try: + for inferred in node.infer(): + # Check to see if this returns a static or a class method. + _type = _infer_decorator_callchain(inferred) + if _type is not None: + return _type + + if not isinstance(inferred, ClassDef): + continue + for ancestor in inferred.ancestors(): + if not isinstance(ancestor, ClassDef): + continue + if ancestor.is_subtype_of("%s.classmethod" % BUILTINS): + return "classmethod" + if ancestor.is_subtype_of("%s.staticmethod" % BUILTINS): + return "staticmethod" + except exceptions.InferenceError: + pass + return type_name + + @decorators_mod.cachedproperty + def fromlineno(self): + """The first line that this node appears on in the source code. + + :type: int or None + """ + # lineno is the line number of the first decorator, we want the def + # statement lineno + lineno = self.lineno + if self.decorators is not None: + lineno += sum( + node.tolineno - node.lineno + 1 for node in self.decorators.nodes + ) + + return lineno + + @decorators_mod.cachedproperty + def blockstart_tolineno(self): + """The line on which the beginning of this block ends. + + :type: int + """ + return self.args.tolineno + + def block_range(self, lineno): + """Get a range from the given line number to where this node ends. + + :param lineno: Unused. + :type lineno: int + + :returns: The range of line numbers that this node belongs to, + :rtype: tuple(int, int) + """ + return self.fromlineno, self.tolineno + + def getattr(self, name, context=None): + """this method doesn't look in the instance_attrs dictionary since it's + done by an Instance proxy at inference time. + """ + if not name: + raise exceptions.AttributeInferenceError( + target=self, attribute=name, context=context + ) + + found_attrs = [] + if name in self.instance_attrs: + found_attrs = self.instance_attrs[name] + if name in self.special_attributes: + found_attrs.append(self.special_attributes.lookup(name)) + if found_attrs: + return found_attrs + raise exceptions.AttributeInferenceError(target=self, attribute=name) + + def igetattr(self, name, context=None): + """Inferred getattr, which returns an iterator of inferred statements.""" + try: + return bases._infer_stmts(self.getattr(name, context), context, frame=self) + except exceptions.AttributeInferenceError as error: + raise exceptions.InferenceError( + error.message, target=self, attribute=name, context=context + ) from error + + def is_method(self): + """Check if this function node represents a method. + + :returns: True if this is a method, False otherwise. + :rtype: bool + """ + # check we are defined in a ClassDef, because this is usually expected + # (e.g. pylint...) when is_method() return True + return self.type != "function" and isinstance(self.parent.frame(), ClassDef) + + @decorators_mod.cached + def decoratornames(self, context=None): + """Get the qualified names of each of the decorators on this function. + + :param context: + An inference context that can be passed to inference functions + :returns: The names of the decorators. + :rtype: set(str) + """ + result = set() + decoratornodes = [] + if self.decorators is not None: + decoratornodes += self.decorators.nodes + decoratornodes += self.extra_decorators + for decnode in decoratornodes: + try: + for infnode in decnode.infer(context=context): + result.add(infnode.qname()) + except exceptions.InferenceError: + continue + return result + + def is_bound(self): + """Check if the function is bound to an instance or class. + + :returns: True if the function is bound to an instance or class, + False otherwise. + :rtype: bool + """ + return self.type == "classmethod" + + def is_abstract(self, pass_is_abstract=True): + """Check if the method is abstract. + + A method is considered abstract if any of the following is true: + * The only statement is 'raise NotImplementedError' + * The only statement is 'pass' and pass_is_abstract is True + * The method is annotated with abc.astractproperty/abc.abstractmethod + + :returns: True if the method is abstract, False otherwise. + :rtype: bool + """ + if self.decorators: + for node in self.decorators.nodes: + try: + inferred = next(node.infer()) + except exceptions.InferenceError: + continue + if inferred and inferred.qname() in ( + "abc.abstractproperty", + "abc.abstractmethod", + ): + return True + + for child_node in self.body: + if isinstance(child_node, node_classes.Raise): + if child_node.raises_not_implemented(): + return True + return pass_is_abstract and isinstance(child_node, node_classes.Pass) + # empty function is the same as function with a single "pass" statement + if pass_is_abstract: + return True + + def is_generator(self): + """Check if this is a generator function. + + :returns: True is this is a generator function, False otherwise. + :rtype: bool + """ + return bool(next(self._get_yield_nodes_skip_lambdas(), False)) + + def infer_call_result(self, caller=None, context=None): + """Infer what the function returns when called. + + :returns: What the function returns. + :rtype: iterable(NodeNG or Uninferable) or None + """ + if self.is_generator(): + if isinstance(self, AsyncFunctionDef): + generator_cls = bases.AsyncGenerator + else: + generator_cls = bases.Generator + result = generator_cls(self) + yield result + return + # This is really a gigantic hack to work around metaclass generators + # that return transient class-generating functions. Pylint's AST structure + # cannot handle a base class object that is only used for calling __new__, + # but does not contribute to the inheritance structure itself. We inject + # a fake class into the hierarchy here for several well-known metaclass + # generators, and filter it out later. + if ( + self.name == "with_metaclass" + and len(self.args.args) == 1 + and self.args.vararg is not None + ): + metaclass = next(caller.args[0].infer(context)) + if isinstance(metaclass, ClassDef): + class_bases = [next(arg.infer(context)) for arg in caller.args[1:]] + new_class = ClassDef(name="temporary_class") + new_class.hide = True + new_class.parent = self + new_class.postinit( + bases=[base for base in class_bases if base != util.Uninferable], + body=[], + decorators=[], + metaclass=metaclass, + ) + yield new_class + return + returns = self._get_return_nodes_skip_functions() + + first_return = next(returns, None) + if not first_return: + if self.body and isinstance(self.body[-1], node_classes.Assert): + yield node_classes.Const(None) + return + + raise exceptions.InferenceError( + "The function does not have any return statements" + ) + + for returnnode in itertools.chain((first_return,), returns): + if returnnode.value is None: + yield node_classes.Const(None) + else: + try: + yield from returnnode.value.infer(context) + except exceptions.InferenceError: + yield util.Uninferable + + def bool_value(self, context=None): + """Determine the boolean value of this node. + + :returns: The boolean value of this node. + For a :class:`FunctionDef` this is always ``True``. + :rtype: bool + """ + return True + + def get_children(self): + if self.decorators is not None: + yield self.decorators + + yield self.args + + if self.returns is not None: + yield self.returns + + yield from self.body + + def scope_lookup(self, node, name, offset=0): + """Lookup where the given name is assigned.""" + if name == "__class__": + # __class__ is an implicit closure reference created by the compiler + # if any methods in a class body refer to either __class__ or super. + # In our case, we want to be able to look it up in the current scope + # when `__class__` is being used. + frame = self.parent.frame() + if isinstance(frame, ClassDef): + return self, [frame] + return super().scope_lookup(node, name, offset) + + +class AsyncFunctionDef(FunctionDef): + """Class representing an :class:`ast.FunctionDef` node. + + A :class:`AsyncFunctionDef` is an asynchronous function + created with the `async` keyword. + + >>> node = astroid.extract_node(''' + async def func(things): + async for thing in things: + print(thing) + ''') + >>> node + + >>> node.body[0] + + """ + + +def _rec_get_names(args, names=None): + """return a list of all argument names""" + if names is None: + names = [] + for arg in args: + if isinstance(arg, node_classes.Tuple): + _rec_get_names(arg.elts, names) + else: + names.append(arg.name) + return names + + +def _is_metaclass(klass, seen=None): + """ Return if the given class can be + used as a metaclass. + """ + if klass.name == "type": + return True + if seen is None: + seen = set() + for base in klass.bases: + try: + for baseobj in base.infer(): + baseobj_name = baseobj.qname() + if baseobj_name in seen: + continue + + seen.add(baseobj_name) + if isinstance(baseobj, bases.Instance): + # not abstract + return False + if baseobj is util.Uninferable: + continue + if baseobj is klass: + continue + if not isinstance(baseobj, ClassDef): + continue + if baseobj._type == "metaclass": + return True + if _is_metaclass(baseobj, seen): + return True + except exceptions.InferenceError: + continue + return False + + +def _class_type(klass, ancestors=None): + """return a ClassDef node type to differ metaclass and exception + from 'regular' classes + """ + # XXX we have to store ancestors in case we have an ancestor loop + if klass._type is not None: + return klass._type + if _is_metaclass(klass): + klass._type = "metaclass" + elif klass.name.endswith("Exception"): + klass._type = "exception" + else: + if ancestors is None: + ancestors = set() + klass_name = klass.qname() + if klass_name in ancestors: + # XXX we are in loop ancestors, and have found no type + klass._type = "class" + return "class" + ancestors.add(klass_name) + for base in klass.ancestors(recurs=False): + name = _class_type(base, ancestors) + if name != "class": + if name == "metaclass" and not _is_metaclass(klass): + # don't propagate it if the current class + # can't be a metaclass + continue + klass._type = base.type + break + if klass._type is None: + klass._type = "class" + return klass._type + + +def get_wrapping_class(node): + """Get the class that wraps the given node. + + We consider that a class wraps a node if the class + is a parent for the said node. + + :returns: The class that wraps the given node + :rtype: ClassDef or None + """ + + klass = node.frame() + while klass is not None and not isinstance(klass, ClassDef): + if klass.parent is None: + klass = None + else: + klass = klass.parent.frame() + return klass + + +class ClassDef(mixins.FilterStmtsMixin, LocalsDictNodeNG, node_classes.Statement): + """Class representing an :class:`ast.ClassDef` node. + + >>> node = astroid.extract_node(''' + class Thing: + def my_meth(self, arg): + return arg + self.offset + ''') + >>> node + + """ + + # some of the attributes below are set by the builder module or + # by a raw factories + + # a dictionary of class instances attributes + _astroid_fields = ("decorators", "bases", "body") # name + + decorators = None + """The decorators that are applied to this class. + + :type: Decorators or None + """ + special_attributes = objectmodel.ClassModel() + """The names of special attributes that this class has. + + :type: objectmodel.ClassModel + """ + + _type = None + _metaclass_hack = False + hide = False + type = property( + _class_type, + doc=( + "The class type for this node.\n\n" + "Possible values are: class, metaclass, exception.\n\n" + ":type: str" + ), + ) + _other_fields = ("name", "doc") + _other_other_fields = ("locals", "_newstyle") + _newstyle = None + + def __init__(self, name=None, doc=None, lineno=None, col_offset=None, parent=None): + """ + :param name: The name of the class. + :type name: str or None + + :param doc: The function's docstring. + :type doc: str or None + + :param lineno: The line that this node appears on in the source code. + :type lineno: int or None + + :param col_offset: The column that this node appears on in the + source code. + :type col_offset: int or None + + :param parent: The parent node in the syntax tree. + :type parent: NodeNG or None + """ + self.instance_attrs = {} + self.locals = {} + """A map of the name of a local variable to the node defining it. + + :type: dict(str, NodeNG) + """ + + self.keywords = [] + """The keywords given to the class definition. + + This is usually for :pep:`3115` style metaclass declaration. + + :type: list(Keyword) or None + """ + + self.bases = [] + """What the class inherits from. + + :type: list(NodeNG) + """ + + self.body = [] + """The contents of the class body. + + :type: list(NodeNG) + """ + + self.name = name + """The name of the class. + + :type name: str or None + """ + + self.doc = doc + """The class' docstring. + + :type doc: str or None + """ + + super().__init__(lineno, col_offset, parent) + if parent is not None: + parent.frame().set_local(name, self) + + for local_name, node in self.implicit_locals(): + self.add_local_node(node, local_name) + + def implicit_parameters(self): + return 1 + + def implicit_locals(self): + """Get implicitly defined class definition locals. + + :returns: the the name and Const pair for each local + :rtype: tuple(tuple(str, node_classes.Const), ...) + """ + locals_ = (("__module__", self.special_attributes.attr___module__),) + # __qualname__ is defined in PEP3155 + locals_ += (("__qualname__", self.special_attributes.attr___qualname__),) + return locals_ + + # pylint: disable=redefined-outer-name + def postinit( + self, bases, body, decorators, newstyle=None, metaclass=None, keywords=None + ): + """Do some setup after initialisation. + + :param bases: What the class inherits from. + :type bases: list(NodeNG) + + :param body: The contents of the class body. + :type body: list(NodeNG) + + :param decorators: The decorators that are applied to this class. + :type decorators: Decorators or None + + :param newstyle: Whether this is a new style class or not. + :type newstyle: bool or None + + :param metaclass: The metaclass of this class. + :type metaclass: NodeNG or None + + :param keywords: The keywords given to the class definition. + :type keywords: list(Keyword) or None + """ + self.keywords = keywords + self.bases = bases + self.body = body + self.decorators = decorators + if newstyle is not None: + self._newstyle = newstyle + if metaclass is not None: + self._metaclass = metaclass + + def _newstyle_impl(self, context=None): + if context is None: + context = contextmod.InferenceContext() + if self._newstyle is not None: + return self._newstyle + for base in self.ancestors(recurs=False, context=context): + if base._newstyle_impl(context): + self._newstyle = True + break + klass = self.declared_metaclass() + # could be any callable, we'd need to infer the result of klass(name, + # bases, dict). punt if it's not a class node. + if klass is not None and isinstance(klass, ClassDef): + self._newstyle = klass._newstyle_impl(context) + if self._newstyle is None: + self._newstyle = False + return self._newstyle + + _newstyle = None + newstyle = property( + _newstyle_impl, + doc=("Whether this is a new style class or not\n\n" ":type: bool or None"), + ) + + @decorators_mod.cachedproperty + def blockstart_tolineno(self): + """The line on which the beginning of this block ends. + + :type: int + """ + if self.bases: + return self.bases[-1].tolineno + + return self.fromlineno + + def block_range(self, lineno): + """Get a range from the given line number to where this node ends. + + :param lineno: Unused. + :type lineno: int + + :returns: The range of line numbers that this node belongs to, + :rtype: tuple(int, int) + """ + return self.fromlineno, self.tolineno + + def pytype(self): + """Get the name of the type that this node represents. + + :returns: The name of the type. + :rtype: str + """ + if self.newstyle: + return "%s.type" % BUILTINS + return "%s.classobj" % BUILTINS + + def display_type(self): + """A human readable type of this node. + + :returns: The type of this node. + :rtype: str + """ + return "Class" + + def callable(self): + """Whether this node defines something that is callable. + + :returns: True if this defines something that is callable, + False otherwise. + For a :class:`ClassDef` this is always ``True``. + :rtype: bool + """ + return True + + def is_subtype_of(self, type_name, context=None): + """Whether this class is a subtype of the given type. + + :param type_name: The name of the type of check against. + :type type_name: str + + :returns: True if this class is a subtype of the given type, + False otherwise. + :rtype: bool + """ + if self.qname() == type_name: + return True + for anc in self.ancestors(context=context): + if anc.qname() == type_name: + return True + return False + + def _infer_type_call(self, caller, context): + name_node = next(caller.args[0].infer(context)) + if isinstance(name_node, node_classes.Const) and isinstance( + name_node.value, str + ): + name = name_node.value + else: + return util.Uninferable + + result = ClassDef(name, None) + + # Get the bases of the class. + class_bases = next(caller.args[1].infer(context)) + if isinstance(class_bases, (node_classes.Tuple, node_classes.List)): + bases = [] + for base in class_bases.itered(): + inferred = next(base.infer(context=context)) + if inferred: + bases.append( + node_classes.EvaluatedObject(original=base, value=inferred) + ) + result.bases = bases + else: + # There is currently no AST node that can represent an 'unknown' + # node (Uninferable is not an AST node), therefore we simply return Uninferable here + # although we know at least the name of the class. + return util.Uninferable + + # Get the members of the class + try: + members = next(caller.args[2].infer(context)) + except exceptions.InferenceError: + members = None + + if members and isinstance(members, node_classes.Dict): + for attr, value in members.items: + if isinstance(attr, node_classes.Const) and isinstance(attr.value, str): + result.locals[attr.value] = [value] + + result.parent = caller.parent + return result + + def infer_call_result(self, caller, context=None): + """infer what a class is returning when called""" + if ( + self.is_subtype_of("%s.type" % (BUILTINS,), context) + and len(caller.args) == 3 + ): + result = self._infer_type_call(caller, context) + yield result + return + + dunder_call = None + try: + metaclass = self.metaclass(context=context) + if metaclass is not None: + dunder_call = next(metaclass.igetattr("__call__", context)) + except exceptions.AttributeInferenceError: + pass + + if dunder_call and dunder_call.qname() != "builtins.type.__call__": + # Call type.__call__ if not set metaclass + # (since type is the default metaclass) + context = contextmod.bind_context_to_node(context, self) + yield from dunder_call.infer_call_result(caller, context) + else: + yield self.instantiate_class() + + def scope_lookup(self, node, name, offset=0): + """Lookup where the given name is assigned. + + :param node: The node to look for assignments up to. + Any assignments after the given node are ignored. + :type node: NodeNG + + :param name: The name to find assignments for. + :type name: str + + :param offset: The line offset to filter statements up to. + :type offset: int + + :returns: This scope node and the list of assignments associated to the + given name according to the scope where it has been found (locals, + globals or builtin). + :rtype: tuple(str, list(NodeNG)) + """ + # If the name looks like a builtin name, just try to look + # into the upper scope of this class. We might have a + # decorator that it's poorly named after a builtin object + # inside this class. + lookup_upper_frame = ( + isinstance(node.parent, node_classes.Decorators) + and name in MANAGER.builtins_module + ) + if ( + any(node == base or base.parent_of(node) for base in self.bases) + or lookup_upper_frame + ): + # Handle the case where we have either a name + # in the bases of a class, which exists before + # the actual definition or the case where we have + # a Getattr node, with that name. + # + # name = ... + # class A(name): + # def name(self): ... + # + # import name + # class A(name.Name): + # def name(self): ... + + frame = self.parent.frame() + # line offset to avoid that class A(A) resolve the ancestor to + # the defined class + offset = -1 + else: + frame = self + return frame._scope_lookup(node, name, offset) + + @property + def basenames(self): + """The names of the parent classes + + Names are given in the order they appear in the class definition. + + :type: list(str) + """ + return [bnode.as_string() for bnode in self.bases] + + def ancestors(self, recurs=True, context=None): + """Iterate over the base classes in prefixed depth first order. + + :param recurs: Whether to recurse or return direct ancestors only. + :type recurs: bool + + :returns: The base classes + :rtype: iterable(NodeNG) + """ + # FIXME: should be possible to choose the resolution order + # FIXME: inference make infinite loops possible here + yielded = {self} + if context is None: + context = contextmod.InferenceContext() + if not self.bases and self.qname() != "builtins.object": + yield builtin_lookup("object")[1][0] + return + + for stmt in self.bases: + with context.restore_path(): + try: + for baseobj in stmt.infer(context): + if not isinstance(baseobj, ClassDef): + if isinstance(baseobj, bases.Instance): + baseobj = baseobj._proxied + else: + continue + if not baseobj.hide: + if baseobj in yielded: + continue + yielded.add(baseobj) + yield baseobj + if not recurs: + continue + for grandpa in baseobj.ancestors(recurs=True, context=context): + if grandpa is self: + # This class is the ancestor of itself. + break + if grandpa in yielded: + continue + yielded.add(grandpa) + yield grandpa + except exceptions.InferenceError: + continue + + def local_attr_ancestors(self, name, context=None): + """Iterate over the parents that define the given name. + + :param name: The name to find definitions for. + :type name: str + + :returns: The parents that define the given name. + :rtype: iterable(NodeNG) + """ + # Look up in the mro if we can. This will result in the + # attribute being looked up just as Python does it. + try: + ancestors = self.mro(context)[1:] + except exceptions.MroError: + # Fallback to use ancestors, we can't determine + # a sane MRO. + ancestors = self.ancestors(context=context) + for astroid in ancestors: + if name in astroid: + yield astroid + + def instance_attr_ancestors(self, name, context=None): + """Iterate over the parents that define the given name as an attribute. + + :param name: The name to find definitions for. + :type name: str + + :returns: The parents that define the given name as + an instance attribute. + :rtype: iterable(NodeNG) + """ + for astroid in self.ancestors(context=context): + if name in astroid.instance_attrs: + yield astroid + + def has_base(self, node): + """Whether this class directly inherits from the given node. + + :param node: The node to check for. + :type node: NodeNG + + :returns: True if this class directly inherits from the given node. + :rtype: bool + """ + return node in self.bases + + def local_attr(self, name, context=None): + """Get the list of assign nodes associated to the given name. + + Assignments are looked for in both this class and in parents. + + :returns: The list of assignments to the given name. + :rtype: list(NodeNG) + + :raises AttributeInferenceError: If no attribute with this name + can be found in this class or parent classes. + """ + result = [] + if name in self.locals: + result = self.locals[name] + else: + class_node = next(self.local_attr_ancestors(name, context), None) + if class_node: + result = class_node.locals[name] + result = [n for n in result if not isinstance(n, node_classes.DelAttr)] + if result: + return result + raise exceptions.AttributeInferenceError( + target=self, attribute=name, context=context + ) + + def instance_attr(self, name, context=None): + """Get the list of nodes associated to the given attribute name. + + Assignments are looked for in both this class and in parents. + + :returns: The list of assignments to the given name. + :rtype: list(NodeNG) + + :raises AttributeInferenceError: If no attribute with this name + can be found in this class or parent classes. + """ + # Return a copy, so we don't modify self.instance_attrs, + # which could lead to infinite loop. + values = list(self.instance_attrs.get(name, [])) + # get all values from parents + for class_node in self.instance_attr_ancestors(name, context): + values += class_node.instance_attrs[name] + values = [n for n in values if not isinstance(n, node_classes.DelAttr)] + if values: + return values + raise exceptions.AttributeInferenceError( + target=self, attribute=name, context=context + ) + + def instantiate_class(self): + """Get an :class:`Instance` of the :class:`ClassDef` node. + + :returns: An :class:`Instance` of the :class:`ClassDef` node, + or self if this is not possible. + :rtype: Instance or ClassDef + """ + try: + if any(cls.name in EXCEPTION_BASE_CLASSES for cls in self.mro()): + # Subclasses of exceptions can be exception instances + return objects.ExceptionInstance(self) + except exceptions.MroError: + pass + return bases.Instance(self) + + def getattr(self, name, context=None, class_context=True): + """Get an attribute from this class, using Python's attribute semantic. + + This method doesn't look in the :attr:`instance_attrs` dictionary + since it is done by an :class:`Instance` proxy at inference time. + It may return an :class:`Uninferable` object if + the attribute has not been + found, but a ``__getattr__`` or ``__getattribute__`` method is defined. + If ``class_context`` is given, then it is considered that the + attribute is accessed from a class context, + e.g. ClassDef.attribute, otherwise it might have been accessed + from an instance as well. If ``class_context`` is used in that + case, then a lookup in the implicit metaclass and the explicit + metaclass will be done. + + :param name: The attribute to look for. + :type name: str + + :param class_context: Whether the attribute can be accessed statically. + :type class_context: bool + + :returns: The attribute. + :rtype: list(NodeNG) + + :raises AttributeInferenceError: If the attribute cannot be inferred. + """ + if not name: + raise exceptions.AttributeInferenceError( + target=self, attribute=name, context=context + ) + + values = self.locals.get(name, []) + if name in self.special_attributes and class_context and not values: + result = [self.special_attributes.lookup(name)] + if name == "__bases__": + # Need special treatment, since they are mutable + # and we need to return all the values. + result += values + return result + + # don't modify the list in self.locals! + values = list(values) + for classnode in self.ancestors(recurs=True, context=context): + values += classnode.locals.get(name, []) + + if class_context: + values += self._metaclass_lookup_attribute(name, context) + + if not values: + raise exceptions.AttributeInferenceError( + target=self, attribute=name, context=context + ) + + # Look for AnnAssigns, which are not attributes in the purest sense. + for value in values: + if isinstance(value, node_classes.AssignName): + stmt = value.statement() + if isinstance(stmt, node_classes.AnnAssign) and stmt.value is None: + raise exceptions.AttributeInferenceError( + target=self, attribute=name, context=context + ) + return values + + def _metaclass_lookup_attribute(self, name, context): + """Search the given name in the implicit and the explicit metaclass.""" + attrs = set() + implicit_meta = self.implicit_metaclass() + context = contextmod.copy_context(context) + metaclass = self.metaclass(context=context) + for cls in {implicit_meta, metaclass}: + if cls and cls != self and isinstance(cls, ClassDef): + cls_attributes = self._get_attribute_from_metaclass(cls, name, context) + attrs.update(set(cls_attributes)) + return attrs + + def _get_attribute_from_metaclass(self, cls, name, context): + try: + attrs = cls.getattr(name, context=context, class_context=True) + except exceptions.AttributeInferenceError: + return + + for attr in bases._infer_stmts(attrs, context, frame=cls): + if not isinstance(attr, FunctionDef): + yield attr + continue + + if isinstance(attr, objects.Property): + yield attr + continue + if attr.type == "classmethod": + # If the method is a classmethod, then it will + # be bound to the metaclass, not to the class + # from where the attribute is retrieved. + # get_wrapping_class could return None, so just + # default to the current class. + frame = get_wrapping_class(attr) or self + yield bases.BoundMethod(attr, frame) + elif attr.type == "staticmethod": + yield attr + else: + yield bases.BoundMethod(attr, self) + + def igetattr(self, name, context=None, class_context=True): + """Infer the possible values of the given variable. + + :param name: The name of the variable to infer. + :type name: str + + :returns: The inferred possible values. + :rtype: iterable(NodeNG or Uninferable) + """ + # set lookup name since this is necessary to infer on import nodes for + # instance + context = contextmod.copy_context(context) + context.lookupname = name + + metaclass = self.declared_metaclass(context=context) + try: + attributes = self.getattr(name, context, class_context=class_context) + # If we have more than one attribute, make sure that those starting from + # the second one are from the same scope. This is to account for modifications + # to the attribute happening *after* the attribute's definition (e.g. AugAssigns on lists) + if len(attributes) > 1: + first_attr, attributes = attributes[0], attributes[1:] + first_scope = first_attr.scope() + attributes = [first_attr] + [ + attr + for attr in attributes + if attr.parent and attr.parent.scope() == first_scope + ] + + for inferred in bases._infer_stmts(attributes, context, frame=self): + # yield Uninferable object instead of descriptors when necessary + if not isinstance(inferred, node_classes.Const) and isinstance( + inferred, bases.Instance + ): + try: + inferred._proxied.getattr("__get__", context) + except exceptions.AttributeInferenceError: + yield inferred + else: + yield util.Uninferable + elif isinstance(inferred, objects.Property): + function = inferred.function + if not class_context: + # Through an instance so we can solve the property + yield from function.infer_call_result( + caller=self, context=context + ) + # If we have a metaclass, we're accessing this attribute through + # the class itself, which means we can solve the property + elif metaclass: + # Resolve a property as long as it is not accessed through + # the class itself. + yield from function.infer_call_result( + caller=self, context=context + ) + else: + yield inferred + else: + yield function_to_method(inferred, self) + except exceptions.AttributeInferenceError as error: + if not name.startswith("__") and self.has_dynamic_getattr(context): + # class handle some dynamic attributes, return a Uninferable object + yield util.Uninferable + else: + raise exceptions.InferenceError( + error.message, target=self, attribute=name, context=context + ) + + def has_dynamic_getattr(self, context=None): + """Check if the class has a custom __getattr__ or __getattribute__. + + If any such method is found and it is not from + builtins, nor from an extension module, then the function + will return True. + + :returns: True if the class has a custom + __getattr__ or __getattribute__, False otherwise. + :rtype: bool + """ + + def _valid_getattr(node): + root = node.root() + return root.name != BUILTINS and getattr(root, "pure_python", None) + + try: + return _valid_getattr(self.getattr("__getattr__", context)[0]) + except exceptions.AttributeInferenceError: + # if self.newstyle: XXX cause an infinite recursion error + try: + getattribute = self.getattr("__getattribute__", context)[0] + return _valid_getattr(getattribute) + except exceptions.AttributeInferenceError: + pass + return False + + def getitem(self, index, context=None): + """Return the inference of a subscript. + + This is basically looking up the method in the metaclass and calling it. + + :returns: The inferred value of a subscript to this class. + :rtype: NodeNG + + :raises AstroidTypeError: If this class does not define a + ``__getitem__`` method. + """ + try: + methods = dunder_lookup.lookup(self, "__getitem__") + except exceptions.AttributeInferenceError as exc: + raise exceptions.AstroidTypeError(node=self, context=context) from exc + + method = methods[0] + + # Create a new callcontext for providing index as an argument. + new_context = contextmod.bind_context_to_node(context, self) + new_context.callcontext = contextmod.CallContext(args=[index]) + + try: + return next(method.infer_call_result(self, new_context)) + except exceptions.InferenceError: + return util.Uninferable + + def methods(self): + """Iterate over all of the method defined in this class and its parents. + + :returns: The methods defined on the class. + :rtype: iterable(FunctionDef) + """ + done = {} + for astroid in itertools.chain(iter((self,)), self.ancestors()): + for meth in astroid.mymethods(): + if meth.name in done: + continue + done[meth.name] = None + yield meth + + def mymethods(self): + """Iterate over all of the method defined in this class only. + + :returns: The methods defined on the class. + :rtype: iterable(FunctionDef) + """ + for member in self.values(): + if isinstance(member, FunctionDef): + yield member + + def implicit_metaclass(self): + """Get the implicit metaclass of the current class. + + For newstyle classes, this will return an instance of builtins.type. + For oldstyle classes, it will simply return None, since there's + no implicit metaclass there. + + :returns: The metaclass. + :rtype: builtins.type or None + """ + if self.newstyle: + return builtin_lookup("type")[1][0] + return None + + _metaclass = None + + def declared_metaclass(self, context=None): + """Return the explicit declared metaclass for the current class. + + An explicit declared metaclass is defined + either by passing the ``metaclass`` keyword argument + in the class definition line (Python 3) or (Python 2) by + having a ``__metaclass__`` class attribute, or if there are + no explicit bases but there is a global ``__metaclass__`` variable. + + :returns: The metaclass of this class, + or None if one could not be found. + :rtype: NodeNG or None + """ + for base in self.bases: + try: + for baseobj in base.infer(context=context): + if isinstance(baseobj, ClassDef) and baseobj.hide: + self._metaclass = baseobj._metaclass + self._metaclass_hack = True + break + except exceptions.InferenceError: + pass + + if self._metaclass: + # Expects this from Py3k TreeRebuilder + try: + return next( + node + for node in self._metaclass.infer(context=context) + if node is not util.Uninferable + ) + except (exceptions.InferenceError, StopIteration): + return None + + return None + + def _find_metaclass(self, seen=None, context=None): + if seen is None: + seen = set() + seen.add(self) + + klass = self.declared_metaclass(context=context) + if klass is None: + for parent in self.ancestors(context=context): + if parent not in seen: + klass = parent._find_metaclass(seen) + if klass is not None: + break + return klass + + def metaclass(self, context=None): + """Get the metaclass of this class. + + If this class does not define explicitly a metaclass, + then the first defined metaclass in ancestors will be used + instead. + + :returns: The metaclass of this class. + :rtype: NodeNG or None + """ + return self._find_metaclass(context=context) + + def has_metaclass_hack(self): + return self._metaclass_hack + + def _islots(self): + """ Return an iterator with the inferred slots. """ + if "__slots__" not in self.locals: + return None + for slots in self.igetattr("__slots__"): + # check if __slots__ is a valid type + for meth in ITER_METHODS: + try: + slots.getattr(meth) + break + except exceptions.AttributeInferenceError: + continue + else: + continue + + if isinstance(slots, node_classes.Const): + # a string. Ignore the following checks, + # but yield the node, only if it has a value + if slots.value: + yield slots + continue + if not hasattr(slots, "itered"): + # we can't obtain the values, maybe a .deque? + continue + + if isinstance(slots, node_classes.Dict): + values = [item[0] for item in slots.items] + else: + values = slots.itered() + if values is util.Uninferable: + continue + if not values: + # Stop the iteration, because the class + # has an empty list of slots. + return values + + for elt in values: + try: + for inferred in elt.infer(): + if inferred is util.Uninferable: + continue + if not isinstance( + inferred, node_classes.Const + ) or not isinstance(inferred.value, str): + continue + if not inferred.value: + continue + yield inferred + except exceptions.InferenceError: + continue + + return None + + def _slots(self): + if not self.newstyle: + raise NotImplementedError( + "The concept of slots is undefined for old-style classes." + ) + + slots = self._islots() + try: + first = next(slots) + except StopIteration as exc: + # The class doesn't have a __slots__ definition or empty slots. + if exc.args and exc.args[0] not in ("", None): + return exc.args[0] + return None + return [first] + list(slots) + + # Cached, because inferring them all the time is expensive + @decorators_mod.cached + def slots(self): + """Get all the slots for this node. + + :returns: The names of slots for this class. + If the class doesn't define any slot, through the ``__slots__`` + variable, then this function will return a None. + Also, it will return None in the case the slots were not inferred. + :rtype: list(str) or None + """ + + def grouped_slots(): + # Not interested in object, since it can't have slots. + for cls in self.mro()[:-1]: + try: + cls_slots = cls._slots() + except NotImplementedError: + continue + if cls_slots is not None: + yield from cls_slots + else: + yield None + + if not self.newstyle: + raise NotImplementedError( + "The concept of slots is undefined for old-style classes." + ) + + slots = list(grouped_slots()) + if not all(slot is not None for slot in slots): + return None + + return sorted(set(slots), key=lambda item: item.value) + + def _inferred_bases(self, context=None): + # Similar with .ancestors, but the difference is when one base is inferred, + # only the first object is wanted. That's because + # we aren't interested in superclasses, as in the following + # example: + # + # class SomeSuperClass(object): pass + # class SomeClass(SomeSuperClass): pass + # class Test(SomeClass): pass + # + # Inferring SomeClass from the Test's bases will give + # us both SomeClass and SomeSuperClass, but we are interested + # only in SomeClass. + + if context is None: + context = contextmod.InferenceContext() + if not self.bases and self.qname() != "builtins.object": + yield builtin_lookup("object")[1][0] + return + + for stmt in self.bases: + try: + baseobj = next(stmt.infer(context=context)) + except exceptions.InferenceError: + continue + if isinstance(baseobj, bases.Instance): + baseobj = baseobj._proxied + if not isinstance(baseobj, ClassDef): + continue + if not baseobj.hide: + yield baseobj + else: + yield from baseobj.bases + + def _compute_mro(self, context=None): + inferred_bases = list(self._inferred_bases(context=context)) + bases_mro = [] + for base in inferred_bases: + if base is self: + continue + + try: + mro = base._compute_mro(context=context) + bases_mro.append(mro) + except NotImplementedError: + # Some classes have in their ancestors both newstyle and + # old style classes. For these we can't retrieve the .mro, + # although in Python it's possible, since the class we are + # currently working is in fact new style. + # So, we fallback to ancestors here. + ancestors = list(base.ancestors(context=context)) + bases_mro.append(ancestors) + + unmerged_mro = [[self]] + bases_mro + [inferred_bases] + unmerged_mro = list(clean_duplicates_mro(unmerged_mro, self, context)) + return _c3_merge(unmerged_mro, self, context) + + def mro(self, context=None) -> List["ClassDef"]: + """Get the method resolution order, using C3 linearization. + + :returns: The list of ancestors, sorted by the mro. + :rtype: list(NodeNG) + :raises DuplicateBasesError: Duplicate bases in the same class base + :raises InconsistentMroError: A class' MRO is inconsistent + """ + return self._compute_mro(context=context) + + def bool_value(self, context=None): + """Determine the boolean value of this node. + + :returns: The boolean value of this node. + For a :class:`ClassDef` this is always ``True``. + :rtype: bool + """ + return True + + def get_children(self): + if self.decorators is not None: + yield self.decorators + + yield from self.bases + yield from self.body + + @decorators_mod.cached + def _get_assign_nodes(self): + children_assign_nodes = ( + child_node._get_assign_nodes() for child_node in self.body + ) + return list(itertools.chain.from_iterable(children_assign_nodes)) diff --git a/WebCrawler/venv/Lib/site-packages/astroid/test_utils.py b/WebCrawler/venv/Lib/site-packages/astroid/test_utils.py new file mode 100644 index 0000000..e22c7a4 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/astroid/test_utils.py @@ -0,0 +1,73 @@ +# Copyright (c) 2013-2014 Google, Inc. +# Copyright (c) 2014 LOGILAB S.A. (Paris, FRANCE) +# Copyright (c) 2015-2016, 2018-2019 Claudiu Popa +# Copyright (c) 2015-2016 Ceridwen +# Copyright (c) 2016 Jakub Wilk +# Copyright (c) 2018 Anthony Sottile + +# Licensed under the LGPL: https://www.gnu.org/licenses/old-licenses/lgpl-2.1.en.html +# For details: https://github.com/PyCQA/astroid/blob/master/COPYING.LESSER + +"""Utility functions for test code that uses astroid ASTs as input.""" +import contextlib +import functools +import sys +import warnings + +import pytest + +from astroid import nodes + + +def require_version(minver=None, maxver=None): + """ Compare version of python interpreter to the given one. Skip the test + if older. + """ + + def parse(string, default=None): + string = string or default + try: + return tuple(int(v) for v in string.split(".")) + except ValueError as exc: + raise ValueError( + "{string} is not a correct version : should be X.Y[.Z].".format( + string=string + ) + ) from exc + + def check_require_version(f): + current = sys.version_info[:3] + if parse(minver, "0") < current <= parse(maxver, "4"): + return f + + str_version = ".".join(str(v) for v in sys.version_info) + + @functools.wraps(f) + def new_f(*args, **kwargs): + if minver is not None: + pytest.skip( + "Needs Python > %s. Current version is %s." % (minver, str_version) + ) + elif maxver is not None: + pytest.skip( + "Needs Python <= %s. Current version is %s." % (maxver, str_version) + ) + + return new_f + + return check_require_version + + +def get_name_node(start_from, name, index=0): + return [n for n in start_from.nodes_of_class(nodes.Name) if n.name == name][index] + + +@contextlib.contextmanager +def enable_warning(warning): + warnings.simplefilter("always", warning) + try: + yield + finally: + # Reset it to default value, so it will take + # into account the values from the -W flag. + warnings.simplefilter("default", warning) diff --git a/WebCrawler/venv/Lib/site-packages/astroid/transforms.py b/WebCrawler/venv/Lib/site-packages/astroid/transforms.py new file mode 100644 index 0000000..e5506cc --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/astroid/transforms.py @@ -0,0 +1,90 @@ +# Copyright (c) 2015-2016, 2018 Claudiu Popa +# Copyright (c) 2016 Ceridwen +# Copyright (c) 2018 Nick Drozd + +# Licensed under the LGPL: https://www.gnu.org/licenses/old-licenses/lgpl-2.1.en.html +# For details: https://github.com/PyCQA/astroid/blob/master/COPYING.LESSER + + +import collections +from functools import lru_cache + + +class TransformVisitor: + """A visitor for handling transforms. + + The standard approach of using it is to call + :meth:`~visit` with an *astroid* module and the class + will take care of the rest, walking the tree and running the + transforms for each encountered node. + """ + + TRANSFORM_MAX_CACHE_SIZE = 10000 + + def __init__(self): + self.transforms = collections.defaultdict(list) + + @lru_cache(maxsize=TRANSFORM_MAX_CACHE_SIZE) + def _transform(self, node): + """Call matching transforms for the given node if any and return the + transformed node. + """ + cls = node.__class__ + if cls not in self.transforms: + # no transform registered for this class of node + return node + + transforms = self.transforms[cls] + for transform_func, predicate in transforms: + if predicate is None or predicate(node): + ret = transform_func(node) + # if the transformation function returns something, it's + # expected to be a replacement for the node + if ret is not None: + node = ret + if ret.__class__ != cls: + # Can no longer apply the rest of the transforms. + break + return node + + def _visit(self, node): + if hasattr(node, "_astroid_fields"): + for name in node._astroid_fields: + value = getattr(node, name) + visited = self._visit_generic(value) + if visited != value: + setattr(node, name, visited) + return self._transform(node) + + def _visit_generic(self, node): + if isinstance(node, list): + return [self._visit_generic(child) for child in node] + if isinstance(node, tuple): + return tuple(self._visit_generic(child) for child in node) + if not node or isinstance(node, str): + return node + + return self._visit(node) + + def register_transform(self, node_class, transform, predicate=None): + """Register `transform(node)` function to be applied on the given + astroid's `node_class` if `predicate` is None or returns true + when called with the node as argument. + + The transform function may return a value which is then used to + substitute the original node in the tree. + """ + self.transforms[node_class].append((transform, predicate)) + + def unregister_transform(self, node_class, transform, predicate=None): + """Unregister the given transform.""" + self.transforms[node_class].remove((transform, predicate)) + + def visit(self, module): + """Walk the given astroid *tree* and transform each encountered node + + Only the nodes which have transforms registered will actually + be replaced or changed. + """ + module.body = [self._visit(child) for child in module.body] + return self._transform(module) diff --git a/WebCrawler/venv/Lib/site-packages/astroid/util.py b/WebCrawler/venv/Lib/site-packages/astroid/util.py new file mode 100644 index 0000000..3ab7561 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/astroid/util.py @@ -0,0 +1,164 @@ +# Copyright (c) 2015-2018 Claudiu Popa +# Copyright (c) 2015-2016 Ceridwen +# Copyright (c) 2018 Bryce Guinta +# Copyright (c) 2018 Nick Drozd + +# Licensed under the LGPL: https://www.gnu.org/licenses/old-licenses/lgpl-2.1.en.html +# For details: https://github.com/PyCQA/astroid/blob/master/COPYING.LESSER + +import warnings +from itertools import islice + +import importlib +import lazy_object_proxy + + +def lazy_descriptor(obj): + class DescriptorProxy(lazy_object_proxy.Proxy): + def __get__(self, instance, owner=None): + return self.__class__.__get__(self, instance) + + return DescriptorProxy(obj) + + +def lazy_import(module_name): + return lazy_object_proxy.Proxy( + lambda: importlib.import_module("." + module_name, "astroid") + ) + + +@object.__new__ +class Uninferable: + """Special inference object, which is returned when inference fails.""" + + def __repr__(self): + return "Uninferable" + + __str__ = __repr__ + + def __getattribute__(self, name): + if name == "next": + raise AttributeError("next method should not be called") + if name.startswith("__") and name.endswith("__"): + return object.__getattribute__(self, name) + if name == "accept": + return object.__getattribute__(self, name) + return self + + def __call__(self, *args, **kwargs): + return self + + def __bool__(self): + return False + + __nonzero__ = __bool__ + + def accept(self, visitor): + func = getattr(visitor, "visit_uninferable") + return func(self) + + +class BadOperationMessage: + """Object which describes a TypeError occurred somewhere in the inference chain + + This is not an exception, but a container object which holds the types and + the error which occurred. + """ + + +class BadUnaryOperationMessage(BadOperationMessage): + """Object which describes operational failures on UnaryOps.""" + + def __init__(self, operand, op, error): + self.operand = operand + self.op = op + self.error = error + + @property + def _object_type_helper(self): + helpers = lazy_import("helpers") + return helpers.object_type + + def _object_type(self, obj): + # pylint: disable=not-callable; can't infer lazy_import + objtype = self._object_type_helper(obj) + if objtype is Uninferable: + return None + + return objtype + + def __str__(self): + if hasattr(self.operand, "name"): + operand_type = self.operand.name + else: + object_type = self._object_type(self.operand) + if hasattr(object_type, "name"): + operand_type = object_type.name + else: + # Just fallback to as_string + operand_type = object_type.as_string() + + msg = "bad operand type for unary {}: {}" + return msg.format(self.op, operand_type) + + +class BadBinaryOperationMessage(BadOperationMessage): + """Object which describes type errors for BinOps.""" + + def __init__(self, left_type, op, right_type): + self.left_type = left_type + self.right_type = right_type + self.op = op + + def __str__(self): + msg = "unsupported operand type(s) for {}: {!r} and {!r}" + return msg.format(self.op, self.left_type.name, self.right_type.name) + + +def _instancecheck(cls, other): + wrapped = cls.__wrapped__ + other_cls = other.__class__ + is_instance_of = wrapped is other_cls or issubclass(other_cls, wrapped) + warnings.warn( + "%r is deprecated and slated for removal in astroid " + "2.0, use %r instead" % (cls.__class__.__name__, wrapped.__name__), + PendingDeprecationWarning, + stacklevel=2, + ) + return is_instance_of + + +def proxy_alias(alias_name, node_type): + """Get a Proxy from the given name to the given node type.""" + proxy = type( + alias_name, + (lazy_object_proxy.Proxy,), + { + "__class__": object.__dict__["__class__"], + "__instancecheck__": _instancecheck, + }, + ) + return proxy(lambda: node_type) + + +def limit_inference(iterator, size): + """Limit inference amount. + + Limit inference amount to help with performance issues with + exponentially exploding possible results. + + :param iterator: Inference generator to limit + :type iterator: Iterator(NodeNG) + + :param size: Maximum mount of nodes yielded plus an + Uninferable at the end if limit reached + :type size: int + + :yields: A possibly modified generator + :rtype param: Iterable + """ + yield from islice(iterator, size) + has_more = next(iterator, False) + if has_more is not False: + yield Uninferable + return diff --git a/WebCrawler/venv/Lib/site-packages/attr/__init__.py b/WebCrawler/venv/Lib/site-packages/attr/__init__.py new file mode 100644 index 0000000..bf329ca --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/attr/__init__.py @@ -0,0 +1,76 @@ +from __future__ import absolute_import, division, print_function + +import sys + +from functools import partial + +from . import converters, exceptions, filters, setters, validators +from ._config import get_run_validators, set_run_validators +from ._funcs import asdict, assoc, astuple, evolve, has, resolve_types +from ._make import ( + NOTHING, + Attribute, + Factory, + attrib, + attrs, + fields, + fields_dict, + make_class, + validate, +) +from ._version_info import VersionInfo + + +__version__ = "20.3.0" +__version_info__ = VersionInfo._from_version_string(__version__) + +__title__ = "attrs" +__description__ = "Classes Without Boilerplate" +__url__ = "https://www.attrs.org/" +__uri__ = __url__ +__doc__ = __description__ + " <" + __uri__ + ">" + +__author__ = "Hynek Schlawack" +__email__ = "hs@ox.cx" + +__license__ = "MIT" +__copyright__ = "Copyright (c) 2015 Hynek Schlawack" + + +s = attributes = attrs +ib = attr = attrib +dataclass = partial(attrs, auto_attribs=True) # happy Easter ;) + +__all__ = [ + "Attribute", + "Factory", + "NOTHING", + "asdict", + "assoc", + "astuple", + "attr", + "attrib", + "attributes", + "attrs", + "converters", + "evolve", + "exceptions", + "fields", + "fields_dict", + "filters", + "get_run_validators", + "has", + "ib", + "make_class", + "resolve_types", + "s", + "set_run_validators", + "setters", + "validate", + "validators", +] + +if sys.version_info[:2] >= (3, 6): + from ._next_gen import define, field, frozen, mutable + + __all__.extend((define, field, frozen, mutable)) diff --git a/WebCrawler/venv/Lib/site-packages/attr/__init__.pyi b/WebCrawler/venv/Lib/site-packages/attr/__init__.pyi new file mode 100644 index 0000000..442d6e7 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/attr/__init__.pyi @@ -0,0 +1,433 @@ +from typing import ( + Any, + Callable, + Dict, + Generic, + List, + Optional, + Sequence, + Mapping, + Tuple, + Type, + TypeVar, + Union, + overload, +) + +# `import X as X` is required to make these public +from . import exceptions as exceptions +from . import filters as filters +from . import converters as converters +from . import setters as setters +from . import validators as validators + +from ._version_info import VersionInfo + +__version__: str +__version_info__: VersionInfo +__title__: str +__description__: str +__url__: str +__uri__: str +__author__: str +__email__: str +__license__: str +__copyright__: str + +_T = TypeVar("_T") +_C = TypeVar("_C", bound=type) + +_ValidatorType = Callable[[Any, Attribute[_T], _T], Any] +_ConverterType = Callable[[Any], Any] +_FilterType = Callable[[Attribute[_T], _T], bool] +_ReprType = Callable[[Any], str] +_ReprArgType = Union[bool, _ReprType] +_OnSetAttrType = Callable[[Any, Attribute[Any], Any], Any] +_OnSetAttrArgType = Union[ + _OnSetAttrType, List[_OnSetAttrType], setters._NoOpType +] +_FieldTransformer = Callable[[type, List[Attribute]], List[Attribute]] +# FIXME: in reality, if multiple validators are passed they must be in a list +# or tuple, but those are invariant and so would prevent subtypes of +# _ValidatorType from working when passed in a list or tuple. +_ValidatorArgType = Union[_ValidatorType[_T], Sequence[_ValidatorType[_T]]] + +# _make -- + +NOTHING: object + +# NOTE: Factory lies about its return type to make this possible: +# `x: List[int] # = Factory(list)` +# Work around mypy issue #4554 in the common case by using an overload. +@overload +def Factory(factory: Callable[[], _T]) -> _T: ... +@overload +def Factory( + factory: Union[Callable[[Any], _T], Callable[[], _T]], + takes_self: bool = ..., +) -> _T: ... + +class Attribute(Generic[_T]): + name: str + default: Optional[_T] + validator: Optional[_ValidatorType[_T]] + repr: _ReprArgType + cmp: bool + eq: bool + order: bool + hash: Optional[bool] + init: bool + converter: Optional[_ConverterType] + metadata: Dict[Any, Any] + type: Optional[Type[_T]] + kw_only: bool + on_setattr: _OnSetAttrType + +# NOTE: We had several choices for the annotation to use for type arg: +# 1) Type[_T] +# - Pros: Handles simple cases correctly +# - Cons: Might produce less informative errors in the case of conflicting +# TypeVars e.g. `attr.ib(default='bad', type=int)` +# 2) Callable[..., _T] +# - Pros: Better error messages than #1 for conflicting TypeVars +# - Cons: Terrible error messages for validator checks. +# e.g. attr.ib(type=int, validator=validate_str) +# -> error: Cannot infer function type argument +# 3) type (and do all of the work in the mypy plugin) +# - Pros: Simple here, and we could customize the plugin with our own errors. +# - Cons: Would need to write mypy plugin code to handle all the cases. +# We chose option #1. + +# `attr` lies about its return type to make the following possible: +# attr() -> Any +# attr(8) -> int +# attr(validator=) -> Whatever the callable expects. +# This makes this type of assignments possible: +# x: int = attr(8) +# +# This form catches explicit None or no default but with no other arguments +# returns Any. +@overload +def attrib( + default: None = ..., + validator: None = ..., + repr: _ReprArgType = ..., + cmp: Optional[bool] = ..., + hash: Optional[bool] = ..., + init: bool = ..., + metadata: Optional[Mapping[Any, Any]] = ..., + type: None = ..., + converter: None = ..., + factory: None = ..., + kw_only: bool = ..., + eq: Optional[bool] = ..., + order: Optional[bool] = ..., + on_setattr: Optional[_OnSetAttrArgType] = ..., +) -> Any: ... + +# This form catches an explicit None or no default and infers the type from the +# other arguments. +@overload +def attrib( + default: None = ..., + validator: Optional[_ValidatorArgType[_T]] = ..., + repr: _ReprArgType = ..., + cmp: Optional[bool] = ..., + hash: Optional[bool] = ..., + init: bool = ..., + metadata: Optional[Mapping[Any, Any]] = ..., + type: Optional[Type[_T]] = ..., + converter: Optional[_ConverterType] = ..., + factory: Optional[Callable[[], _T]] = ..., + kw_only: bool = ..., + eq: Optional[bool] = ..., + order: Optional[bool] = ..., + on_setattr: Optional[_OnSetAttrArgType] = ..., +) -> _T: ... + +# This form catches an explicit default argument. +@overload +def attrib( + default: _T, + validator: Optional[_ValidatorArgType[_T]] = ..., + repr: _ReprArgType = ..., + cmp: Optional[bool] = ..., + hash: Optional[bool] = ..., + init: bool = ..., + metadata: Optional[Mapping[Any, Any]] = ..., + type: Optional[Type[_T]] = ..., + converter: Optional[_ConverterType] = ..., + factory: Optional[Callable[[], _T]] = ..., + kw_only: bool = ..., + eq: Optional[bool] = ..., + order: Optional[bool] = ..., + on_setattr: Optional[_OnSetAttrArgType] = ..., +) -> _T: ... + +# This form covers type=non-Type: e.g. forward references (str), Any +@overload +def attrib( + default: Optional[_T] = ..., + validator: Optional[_ValidatorArgType[_T]] = ..., + repr: _ReprArgType = ..., + cmp: Optional[bool] = ..., + hash: Optional[bool] = ..., + init: bool = ..., + metadata: Optional[Mapping[Any, Any]] = ..., + type: object = ..., + converter: Optional[_ConverterType] = ..., + factory: Optional[Callable[[], _T]] = ..., + kw_only: bool = ..., + eq: Optional[bool] = ..., + order: Optional[bool] = ..., + on_setattr: Optional[_OnSetAttrArgType] = ..., +) -> Any: ... +@overload +def field( + *, + default: None = ..., + validator: None = ..., + repr: _ReprArgType = ..., + hash: Optional[bool] = ..., + init: bool = ..., + metadata: Optional[Mapping[Any, Any]] = ..., + converter: None = ..., + factory: None = ..., + kw_only: bool = ..., + eq: Optional[bool] = ..., + order: Optional[bool] = ..., + on_setattr: Optional[_OnSetAttrArgType] = ..., +) -> Any: ... + +# This form catches an explicit None or no default and infers the type from the +# other arguments. +@overload +def field( + *, + default: None = ..., + validator: Optional[_ValidatorArgType[_T]] = ..., + repr: _ReprArgType = ..., + hash: Optional[bool] = ..., + init: bool = ..., + metadata: Optional[Mapping[Any, Any]] = ..., + converter: Optional[_ConverterType] = ..., + factory: Optional[Callable[[], _T]] = ..., + kw_only: bool = ..., + eq: Optional[bool] = ..., + order: Optional[bool] = ..., + on_setattr: Optional[_OnSetAttrArgType] = ..., +) -> _T: ... + +# This form catches an explicit default argument. +@overload +def field( + *, + default: _T, + validator: Optional[_ValidatorArgType[_T]] = ..., + repr: _ReprArgType = ..., + hash: Optional[bool] = ..., + init: bool = ..., + metadata: Optional[Mapping[Any, Any]] = ..., + converter: Optional[_ConverterType] = ..., + factory: Optional[Callable[[], _T]] = ..., + kw_only: bool = ..., + eq: Optional[bool] = ..., + order: Optional[bool] = ..., + on_setattr: Optional[_OnSetAttrArgType] = ..., +) -> _T: ... + +# This form covers type=non-Type: e.g. forward references (str), Any +@overload +def field( + *, + default: Optional[_T] = ..., + validator: Optional[_ValidatorArgType[_T]] = ..., + repr: _ReprArgType = ..., + hash: Optional[bool] = ..., + init: bool = ..., + metadata: Optional[Mapping[Any, Any]] = ..., + converter: Optional[_ConverterType] = ..., + factory: Optional[Callable[[], _T]] = ..., + kw_only: bool = ..., + eq: Optional[bool] = ..., + order: Optional[bool] = ..., + on_setattr: Optional[_OnSetAttrArgType] = ..., +) -> Any: ... +@overload +def attrs( + maybe_cls: _C, + these: Optional[Dict[str, Any]] = ..., + repr_ns: Optional[str] = ..., + repr: bool = ..., + cmp: Optional[bool] = ..., + hash: Optional[bool] = ..., + init: bool = ..., + slots: bool = ..., + frozen: bool = ..., + weakref_slot: bool = ..., + str: bool = ..., + auto_attribs: bool = ..., + kw_only: bool = ..., + cache_hash: bool = ..., + auto_exc: bool = ..., + eq: Optional[bool] = ..., + order: Optional[bool] = ..., + auto_detect: bool = ..., + collect_by_mro: bool = ..., + getstate_setstate: Optional[bool] = ..., + on_setattr: Optional[_OnSetAttrArgType] = ..., + field_transformer: Optional[_FieldTransformer] = ..., +) -> _C: ... +@overload +def attrs( + maybe_cls: None = ..., + these: Optional[Dict[str, Any]] = ..., + repr_ns: Optional[str] = ..., + repr: bool = ..., + cmp: Optional[bool] = ..., + hash: Optional[bool] = ..., + init: bool = ..., + slots: bool = ..., + frozen: bool = ..., + weakref_slot: bool = ..., + str: bool = ..., + auto_attribs: bool = ..., + kw_only: bool = ..., + cache_hash: bool = ..., + auto_exc: bool = ..., + eq: Optional[bool] = ..., + order: Optional[bool] = ..., + auto_detect: bool = ..., + collect_by_mro: bool = ..., + getstate_setstate: Optional[bool] = ..., + on_setattr: Optional[_OnSetAttrArgType] = ..., + field_transformer: Optional[_FieldTransformer] = ..., +) -> Callable[[_C], _C]: ... +@overload +def define( + maybe_cls: _C, + *, + these: Optional[Dict[str, Any]] = ..., + repr: bool = ..., + hash: Optional[bool] = ..., + init: bool = ..., + slots: bool = ..., + frozen: bool = ..., + weakref_slot: bool = ..., + str: bool = ..., + auto_attribs: bool = ..., + kw_only: bool = ..., + cache_hash: bool = ..., + auto_exc: bool = ..., + eq: Optional[bool] = ..., + order: Optional[bool] = ..., + auto_detect: bool = ..., + getstate_setstate: Optional[bool] = ..., + on_setattr: Optional[_OnSetAttrArgType] = ..., + field_transformer: Optional[_FieldTransformer] = ..., +) -> _C: ... +@overload +def define( + maybe_cls: None = ..., + *, + these: Optional[Dict[str, Any]] = ..., + repr: bool = ..., + hash: Optional[bool] = ..., + init: bool = ..., + slots: bool = ..., + frozen: bool = ..., + weakref_slot: bool = ..., + str: bool = ..., + auto_attribs: bool = ..., + kw_only: bool = ..., + cache_hash: bool = ..., + auto_exc: bool = ..., + eq: Optional[bool] = ..., + order: Optional[bool] = ..., + auto_detect: bool = ..., + getstate_setstate: Optional[bool] = ..., + on_setattr: Optional[_OnSetAttrArgType] = ..., + field_transformer: Optional[_FieldTransformer] = ..., +) -> Callable[[_C], _C]: ... + +mutable = define +frozen = define # they differ only in their defaults + +# TODO: add support for returning NamedTuple from the mypy plugin +class _Fields(Tuple[Attribute[Any], ...]): + def __getattr__(self, name: str) -> Attribute[Any]: ... + +def fields(cls: type) -> _Fields: ... +def fields_dict(cls: type) -> Dict[str, Attribute[Any]]: ... +def validate(inst: Any) -> None: ... +def resolve_types( + cls: _C, + globalns: Optional[Dict[str, Any]] = ..., + localns: Optional[Dict[str, Any]] = ..., +) -> _C: ... + +# TODO: add support for returning a proper attrs class from the mypy plugin +# we use Any instead of _CountingAttr so that e.g. `make_class('Foo', +# [attr.ib()])` is valid +def make_class( + name: str, + attrs: Union[List[str], Tuple[str, ...], Dict[str, Any]], + bases: Tuple[type, ...] = ..., + repr_ns: Optional[str] = ..., + repr: bool = ..., + cmp: Optional[bool] = ..., + hash: Optional[bool] = ..., + init: bool = ..., + slots: bool = ..., + frozen: bool = ..., + weakref_slot: bool = ..., + str: bool = ..., + auto_attribs: bool = ..., + kw_only: bool = ..., + cache_hash: bool = ..., + auto_exc: bool = ..., + eq: Optional[bool] = ..., + order: Optional[bool] = ..., + collect_by_mro: bool = ..., + on_setattr: Optional[_OnSetAttrArgType] = ..., + field_transformer: Optional[_FieldTransformer] = ..., +) -> type: ... + +# _funcs -- + +# TODO: add support for returning TypedDict from the mypy plugin +# FIXME: asdict/astuple do not honor their factory args. Waiting on one of +# these: +# https://github.com/python/mypy/issues/4236 +# https://github.com/python/typing/issues/253 +def asdict( + inst: Any, + recurse: bool = ..., + filter: Optional[_FilterType[Any]] = ..., + dict_factory: Type[Mapping[Any, Any]] = ..., + retain_collection_types: bool = ..., + value_serializer: Optional[Callable[[type, Attribute, Any], Any]] = ..., +) -> Dict[str, Any]: ... + +# TODO: add support for returning NamedTuple from the mypy plugin +def astuple( + inst: Any, + recurse: bool = ..., + filter: Optional[_FilterType[Any]] = ..., + tuple_factory: Type[Sequence[Any]] = ..., + retain_collection_types: bool = ..., +) -> Tuple[Any, ...]: ... +def has(cls: type) -> bool: ... +def assoc(inst: _T, **changes: Any) -> _T: ... +def evolve(inst: _T, **changes: Any) -> _T: ... + +# _config -- + +def set_run_validators(run: bool) -> None: ... +def get_run_validators() -> bool: ... + +# aliases -- + +s = attributes = attrs +ib = attr = attrib +dataclass = attrs # Technically, partial(attrs, auto_attribs=True) ;) diff --git a/WebCrawler/venv/Lib/site-packages/attr/__pycache__/__init__.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/attr/__pycache__/__init__.cpython-39.pyc new file mode 100644 index 0000000..c264ae3 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/attr/__pycache__/__init__.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/attr/__pycache__/_compat.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/attr/__pycache__/_compat.cpython-39.pyc new file mode 100644 index 0000000..1392596 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/attr/__pycache__/_compat.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/attr/__pycache__/_config.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/attr/__pycache__/_config.cpython-39.pyc new file mode 100644 index 0000000..1bbf95f Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/attr/__pycache__/_config.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/attr/__pycache__/_funcs.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/attr/__pycache__/_funcs.cpython-39.pyc new file mode 100644 index 0000000..07346a4 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/attr/__pycache__/_funcs.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/attr/__pycache__/_make.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/attr/__pycache__/_make.cpython-39.pyc new file mode 100644 index 0000000..ffd0e5a Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/attr/__pycache__/_make.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/attr/__pycache__/_next_gen.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/attr/__pycache__/_next_gen.cpython-39.pyc new file mode 100644 index 0000000..7064812 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/attr/__pycache__/_next_gen.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/attr/__pycache__/_version_info.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/attr/__pycache__/_version_info.cpython-39.pyc new file mode 100644 index 0000000..0c94039 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/attr/__pycache__/_version_info.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/attr/__pycache__/converters.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/attr/__pycache__/converters.cpython-39.pyc new file mode 100644 index 0000000..b34fa74 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/attr/__pycache__/converters.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/attr/__pycache__/exceptions.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/attr/__pycache__/exceptions.cpython-39.pyc new file mode 100644 index 0000000..e8c0b0d Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/attr/__pycache__/exceptions.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/attr/__pycache__/filters.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/attr/__pycache__/filters.cpython-39.pyc new file mode 100644 index 0000000..f34a2aa Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/attr/__pycache__/filters.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/attr/__pycache__/setters.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/attr/__pycache__/setters.cpython-39.pyc new file mode 100644 index 0000000..c0f66f9 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/attr/__pycache__/setters.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/attr/__pycache__/validators.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/attr/__pycache__/validators.cpython-39.pyc new file mode 100644 index 0000000..773e5cc Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/attr/__pycache__/validators.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/attr/_compat.py b/WebCrawler/venv/Lib/site-packages/attr/_compat.py new file mode 100644 index 0000000..b0ead6e --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/attr/_compat.py @@ -0,0 +1,231 @@ +from __future__ import absolute_import, division, print_function + +import platform +import sys +import types +import warnings + + +PY2 = sys.version_info[0] == 2 +PYPY = platform.python_implementation() == "PyPy" + + +if PYPY or sys.version_info[:2] >= (3, 6): + ordered_dict = dict +else: + from collections import OrderedDict + + ordered_dict = OrderedDict + + +if PY2: + from collections import Mapping, Sequence + + from UserDict import IterableUserDict + + # We 'bundle' isclass instead of using inspect as importing inspect is + # fairly expensive (order of 10-15 ms for a modern machine in 2016) + def isclass(klass): + return isinstance(klass, (type, types.ClassType)) + + # TYPE is used in exceptions, repr(int) is different on Python 2 and 3. + TYPE = "type" + + def iteritems(d): + return d.iteritems() + + # Python 2 is bereft of a read-only dict proxy, so we make one! + class ReadOnlyDict(IterableUserDict): + """ + Best-effort read-only dict wrapper. + """ + + def __setitem__(self, key, val): + # We gently pretend we're a Python 3 mappingproxy. + raise TypeError( + "'mappingproxy' object does not support item assignment" + ) + + def update(self, _): + # We gently pretend we're a Python 3 mappingproxy. + raise AttributeError( + "'mappingproxy' object has no attribute 'update'" + ) + + def __delitem__(self, _): + # We gently pretend we're a Python 3 mappingproxy. + raise TypeError( + "'mappingproxy' object does not support item deletion" + ) + + def clear(self): + # We gently pretend we're a Python 3 mappingproxy. + raise AttributeError( + "'mappingproxy' object has no attribute 'clear'" + ) + + def pop(self, key, default=None): + # We gently pretend we're a Python 3 mappingproxy. + raise AttributeError( + "'mappingproxy' object has no attribute 'pop'" + ) + + def popitem(self): + # We gently pretend we're a Python 3 mappingproxy. + raise AttributeError( + "'mappingproxy' object has no attribute 'popitem'" + ) + + def setdefault(self, key, default=None): + # We gently pretend we're a Python 3 mappingproxy. + raise AttributeError( + "'mappingproxy' object has no attribute 'setdefault'" + ) + + def __repr__(self): + # Override to be identical to the Python 3 version. + return "mappingproxy(" + repr(self.data) + ")" + + def metadata_proxy(d): + res = ReadOnlyDict() + res.data.update(d) # We blocked update, so we have to do it like this. + return res + + def just_warn(*args, **kw): # pragma: no cover + """ + We only warn on Python 3 because we are not aware of any concrete + consequences of not setting the cell on Python 2. + """ + + +else: # Python 3 and later. + from collections.abc import Mapping, Sequence # noqa + + def just_warn(*args, **kw): + """ + We only warn on Python 3 because we are not aware of any concrete + consequences of not setting the cell on Python 2. + """ + warnings.warn( + "Running interpreter doesn't sufficiently support code object " + "introspection. Some features like bare super() or accessing " + "__class__ will not work with slotted classes.", + RuntimeWarning, + stacklevel=2, + ) + + def isclass(klass): + return isinstance(klass, type) + + TYPE = "class" + + def iteritems(d): + return d.items() + + def metadata_proxy(d): + return types.MappingProxyType(dict(d)) + + +def make_set_closure_cell(): + """Return a function of two arguments (cell, value) which sets + the value stored in the closure cell `cell` to `value`. + """ + # pypy makes this easy. (It also supports the logic below, but + # why not do the easy/fast thing?) + if PYPY: + + def set_closure_cell(cell, value): + cell.__setstate__((value,)) + + return set_closure_cell + + # Otherwise gotta do it the hard way. + + # Create a function that will set its first cellvar to `value`. + def set_first_cellvar_to(value): + x = value + return + + # This function will be eliminated as dead code, but + # not before its reference to `x` forces `x` to be + # represented as a closure cell rather than a local. + def force_x_to_be_a_cell(): # pragma: no cover + return x + + try: + # Extract the code object and make sure our assumptions about + # the closure behavior are correct. + if PY2: + co = set_first_cellvar_to.func_code + else: + co = set_first_cellvar_to.__code__ + if co.co_cellvars != ("x",) or co.co_freevars != (): + raise AssertionError # pragma: no cover + + # Convert this code object to a code object that sets the + # function's first _freevar_ (not cellvar) to the argument. + if sys.version_info >= (3, 8): + # CPython 3.8+ has an incompatible CodeType signature + # (added a posonlyargcount argument) but also added + # CodeType.replace() to do this without counting parameters. + set_first_freevar_code = co.replace( + co_cellvars=co.co_freevars, co_freevars=co.co_cellvars + ) + else: + args = [co.co_argcount] + if not PY2: + args.append(co.co_kwonlyargcount) + args.extend( + [ + co.co_nlocals, + co.co_stacksize, + co.co_flags, + co.co_code, + co.co_consts, + co.co_names, + co.co_varnames, + co.co_filename, + co.co_name, + co.co_firstlineno, + co.co_lnotab, + # These two arguments are reversed: + co.co_cellvars, + co.co_freevars, + ] + ) + set_first_freevar_code = types.CodeType(*args) + + def set_closure_cell(cell, value): + # Create a function using the set_first_freevar_code, + # whose first closure cell is `cell`. Calling it will + # change the value of that cell. + setter = types.FunctionType( + set_first_freevar_code, {}, "setter", (), (cell,) + ) + # And call it to set the cell. + setter(value) + + # Make sure it works on this interpreter: + def make_func_with_cell(): + x = None + + def func(): + return x # pragma: no cover + + return func + + if PY2: + cell = make_func_with_cell().func_closure[0] + else: + cell = make_func_with_cell().__closure__[0] + set_closure_cell(cell, 100) + if cell.cell_contents != 100: + raise AssertionError # pragma: no cover + + except Exception: + return just_warn + else: + return set_closure_cell + + +set_closure_cell = make_set_closure_cell() diff --git a/WebCrawler/venv/Lib/site-packages/attr/_config.py b/WebCrawler/venv/Lib/site-packages/attr/_config.py new file mode 100644 index 0000000..8ec9209 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/attr/_config.py @@ -0,0 +1,23 @@ +from __future__ import absolute_import, division, print_function + + +__all__ = ["set_run_validators", "get_run_validators"] + +_run_validators = True + + +def set_run_validators(run): + """ + Set whether or not validators are run. By default, they are run. + """ + if not isinstance(run, bool): + raise TypeError("'run' must be bool.") + global _run_validators + _run_validators = run + + +def get_run_validators(): + """ + Return whether or not validators are run. + """ + return _run_validators diff --git a/WebCrawler/venv/Lib/site-packages/attr/_funcs.py b/WebCrawler/venv/Lib/site-packages/attr/_funcs.py new file mode 100644 index 0000000..e6c930c --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/attr/_funcs.py @@ -0,0 +1,390 @@ +from __future__ import absolute_import, division, print_function + +import copy + +from ._compat import iteritems +from ._make import NOTHING, _obj_setattr, fields +from .exceptions import AttrsAttributeNotFoundError + + +def asdict( + inst, + recurse=True, + filter=None, + dict_factory=dict, + retain_collection_types=False, + value_serializer=None, +): + """ + Return the ``attrs`` attribute values of *inst* as a dict. + + Optionally recurse into other ``attrs``-decorated classes. + + :param inst: Instance of an ``attrs``-decorated class. + :param bool recurse: Recurse into classes that are also + ``attrs``-decorated. + :param callable filter: A callable whose return code determines whether an + attribute or element is included (``True``) or dropped (``False``). Is + called with the `attr.Attribute` as the first argument and the + value as the second argument. + :param callable dict_factory: A callable to produce dictionaries from. For + example, to produce ordered dictionaries instead of normal Python + dictionaries, pass in ``collections.OrderedDict``. + :param bool retain_collection_types: Do not convert to ``list`` when + encountering an attribute whose type is ``tuple`` or ``set``. Only + meaningful if ``recurse`` is ``True``. + :param Optional[callable] value_serializer: A hook that is called for every + attribute or dict key/value. It receives the current instance, field + and value and must return the (updated) value. The hook is run *after* + the optional *filter* has been applied. + + :rtype: return type of *dict_factory* + + :raise attr.exceptions.NotAnAttrsClassError: If *cls* is not an ``attrs`` + class. + + .. versionadded:: 16.0.0 *dict_factory* + .. versionadded:: 16.1.0 *retain_collection_types* + .. versionadded:: 20.3.0 *value_serializer* + """ + attrs = fields(inst.__class__) + rv = dict_factory() + for a in attrs: + v = getattr(inst, a.name) + if filter is not None and not filter(a, v): + continue + + if value_serializer is not None: + v = value_serializer(inst, a, v) + + if recurse is True: + if has(v.__class__): + rv[a.name] = asdict( + v, + True, + filter, + dict_factory, + retain_collection_types, + value_serializer, + ) + elif isinstance(v, (tuple, list, set, frozenset)): + cf = v.__class__ if retain_collection_types is True else list + rv[a.name] = cf( + [ + _asdict_anything( + i, + filter, + dict_factory, + retain_collection_types, + value_serializer, + ) + for i in v + ] + ) + elif isinstance(v, dict): + df = dict_factory + rv[a.name] = df( + ( + _asdict_anything( + kk, + filter, + df, + retain_collection_types, + value_serializer, + ), + _asdict_anything( + vv, + filter, + df, + retain_collection_types, + value_serializer, + ), + ) + for kk, vv in iteritems(v) + ) + else: + rv[a.name] = v + else: + rv[a.name] = v + return rv + + +def _asdict_anything( + val, + filter, + dict_factory, + retain_collection_types, + value_serializer, +): + """ + ``asdict`` only works on attrs instances, this works on anything. + """ + if getattr(val.__class__, "__attrs_attrs__", None) is not None: + # Attrs class. + rv = asdict( + val, + True, + filter, + dict_factory, + retain_collection_types, + value_serializer, + ) + elif isinstance(val, (tuple, list, set, frozenset)): + cf = val.__class__ if retain_collection_types is True else list + rv = cf( + [ + _asdict_anything( + i, + filter, + dict_factory, + retain_collection_types, + value_serializer, + ) + for i in val + ] + ) + elif isinstance(val, dict): + df = dict_factory + rv = df( + ( + _asdict_anything( + kk, filter, df, retain_collection_types, value_serializer + ), + _asdict_anything( + vv, filter, df, retain_collection_types, value_serializer + ), + ) + for kk, vv in iteritems(val) + ) + else: + rv = val + if value_serializer is not None: + rv = value_serializer(None, None, rv) + + return rv + + +def astuple( + inst, + recurse=True, + filter=None, + tuple_factory=tuple, + retain_collection_types=False, +): + """ + Return the ``attrs`` attribute values of *inst* as a tuple. + + Optionally recurse into other ``attrs``-decorated classes. + + :param inst: Instance of an ``attrs``-decorated class. + :param bool recurse: Recurse into classes that are also + ``attrs``-decorated. + :param callable filter: A callable whose return code determines whether an + attribute or element is included (``True``) or dropped (``False``). Is + called with the `attr.Attribute` as the first argument and the + value as the second argument. + :param callable tuple_factory: A callable to produce tuples from. For + example, to produce lists instead of tuples. + :param bool retain_collection_types: Do not convert to ``list`` + or ``dict`` when encountering an attribute which type is + ``tuple``, ``dict`` or ``set``. Only meaningful if ``recurse`` is + ``True``. + + :rtype: return type of *tuple_factory* + + :raise attr.exceptions.NotAnAttrsClassError: If *cls* is not an ``attrs`` + class. + + .. versionadded:: 16.2.0 + """ + attrs = fields(inst.__class__) + rv = [] + retain = retain_collection_types # Very long. :/ + for a in attrs: + v = getattr(inst, a.name) + if filter is not None and not filter(a, v): + continue + if recurse is True: + if has(v.__class__): + rv.append( + astuple( + v, + recurse=True, + filter=filter, + tuple_factory=tuple_factory, + retain_collection_types=retain, + ) + ) + elif isinstance(v, (tuple, list, set, frozenset)): + cf = v.__class__ if retain is True else list + rv.append( + cf( + [ + astuple( + j, + recurse=True, + filter=filter, + tuple_factory=tuple_factory, + retain_collection_types=retain, + ) + if has(j.__class__) + else j + for j in v + ] + ) + ) + elif isinstance(v, dict): + df = v.__class__ if retain is True else dict + rv.append( + df( + ( + astuple( + kk, + tuple_factory=tuple_factory, + retain_collection_types=retain, + ) + if has(kk.__class__) + else kk, + astuple( + vv, + tuple_factory=tuple_factory, + retain_collection_types=retain, + ) + if has(vv.__class__) + else vv, + ) + for kk, vv in iteritems(v) + ) + ) + else: + rv.append(v) + else: + rv.append(v) + + return rv if tuple_factory is list else tuple_factory(rv) + + +def has(cls): + """ + Check whether *cls* is a class with ``attrs`` attributes. + + :param type cls: Class to introspect. + :raise TypeError: If *cls* is not a class. + + :rtype: bool + """ + return getattr(cls, "__attrs_attrs__", None) is not None + + +def assoc(inst, **changes): + """ + Copy *inst* and apply *changes*. + + :param inst: Instance of a class with ``attrs`` attributes. + :param changes: Keyword changes in the new copy. + + :return: A copy of inst with *changes* incorporated. + + :raise attr.exceptions.AttrsAttributeNotFoundError: If *attr_name* couldn't + be found on *cls*. + :raise attr.exceptions.NotAnAttrsClassError: If *cls* is not an ``attrs`` + class. + + .. deprecated:: 17.1.0 + Use `evolve` instead. + """ + import warnings + + warnings.warn( + "assoc is deprecated and will be removed after 2018/01.", + DeprecationWarning, + stacklevel=2, + ) + new = copy.copy(inst) + attrs = fields(inst.__class__) + for k, v in iteritems(changes): + a = getattr(attrs, k, NOTHING) + if a is NOTHING: + raise AttrsAttributeNotFoundError( + "{k} is not an attrs attribute on {cl}.".format( + k=k, cl=new.__class__ + ) + ) + _obj_setattr(new, k, v) + return new + + +def evolve(inst, **changes): + """ + Create a new instance, based on *inst* with *changes* applied. + + :param inst: Instance of a class with ``attrs`` attributes. + :param changes: Keyword changes in the new copy. + + :return: A copy of inst with *changes* incorporated. + + :raise TypeError: If *attr_name* couldn't be found in the class + ``__init__``. + :raise attr.exceptions.NotAnAttrsClassError: If *cls* is not an ``attrs`` + class. + + .. versionadded:: 17.1.0 + """ + cls = inst.__class__ + attrs = fields(cls) + for a in attrs: + if not a.init: + continue + attr_name = a.name # To deal with private attributes. + init_name = attr_name if attr_name[0] != "_" else attr_name[1:] + if init_name not in changes: + changes[init_name] = getattr(inst, attr_name) + + return cls(**changes) + + +def resolve_types(cls, globalns=None, localns=None): + """ + Resolve any strings and forward annotations in type annotations. + + This is only required if you need concrete types in `Attribute`'s *type* + field. In other words, you don't need to resolve your types if you only + use them for static type checking. + + With no arguments, names will be looked up in the module in which the class + was created. If this is not what you want, e.g. if the name only exists + inside a method, you may pass *globalns* or *localns* to specify other + dictionaries in which to look up these names. See the docs of + `typing.get_type_hints` for more details. + + :param type cls: Class to resolve. + :param Optional[dict] globalns: Dictionary containing global variables. + :param Optional[dict] localns: Dictionary containing local variables. + + :raise TypeError: If *cls* is not a class. + :raise attr.exceptions.NotAnAttrsClassError: If *cls* is not an ``attrs`` + class. + :raise NameError: If types cannot be resolved because of missing variables. + + :returns: *cls* so you can use this function also as a class decorator. + Please note that you have to apply it **after** `attr.s`. That means + the decorator has to come in the line **before** `attr.s`. + + .. versionadded:: 20.1.0 + """ + try: + # Since calling get_type_hints is expensive we cache whether we've + # done it already. + cls.__attrs_types_resolved__ + except AttributeError: + import typing + + hints = typing.get_type_hints(cls, globalns=globalns, localns=localns) + for field in fields(cls): + if field.name in hints: + # Since fields have been frozen we must work around it. + _obj_setattr(field, "type", hints[field.name]) + cls.__attrs_types_resolved__ = True + + # Return the class so you can use it as a decorator too. + return cls diff --git a/WebCrawler/venv/Lib/site-packages/attr/_make.py b/WebCrawler/venv/Lib/site-packages/attr/_make.py new file mode 100644 index 0000000..49484f9 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/attr/_make.py @@ -0,0 +1,2765 @@ +from __future__ import absolute_import, division, print_function + +import copy +import linecache +import sys +import threading +import uuid +import warnings + +from operator import itemgetter + +from . import _config, setters +from ._compat import ( + PY2, + PYPY, + isclass, + iteritems, + metadata_proxy, + ordered_dict, + set_closure_cell, +) +from .exceptions import ( + DefaultAlreadySetError, + FrozenInstanceError, + NotAnAttrsClassError, + PythonTooOldError, + UnannotatedAttributeError, +) + + +# This is used at least twice, so cache it here. +_obj_setattr = object.__setattr__ +_init_converter_pat = "__attr_converter_%s" +_init_factory_pat = "__attr_factory_{}" +_tuple_property_pat = ( + " {attr_name} = _attrs_property(_attrs_itemgetter({index}))" +) +_classvar_prefixes = ("typing.ClassVar", "t.ClassVar", "ClassVar") +# we don't use a double-underscore prefix because that triggers +# name mangling when trying to create a slot for the field +# (when slots=True) +_hash_cache_field = "_attrs_cached_hash" + +_empty_metadata_singleton = metadata_proxy({}) + +# Unique object for unequivocal getattr() defaults. +_sentinel = object() + + +class _Nothing(object): + """ + Sentinel class to indicate the lack of a value when ``None`` is ambiguous. + + ``_Nothing`` is a singleton. There is only ever one of it. + """ + + _singleton = None + + def __new__(cls): + if _Nothing._singleton is None: + _Nothing._singleton = super(_Nothing, cls).__new__(cls) + return _Nothing._singleton + + def __repr__(self): + return "NOTHING" + + +NOTHING = _Nothing() +""" +Sentinel to indicate the lack of a value when ``None`` is ambiguous. +""" + + +class _CacheHashWrapper(int): + """ + An integer subclass that pickles / copies as None + + This is used for non-slots classes with ``cache_hash=True``, to avoid + serializing a potentially (even likely) invalid hash value. Since ``None`` + is the default value for uncalculated hashes, whenever this is copied, + the copy's value for the hash should automatically reset. + + See GH #613 for more details. + """ + + if PY2: + # For some reason `type(None)` isn't callable in Python 2, but we don't + # actually need a constructor for None objects, we just need any + # available function that returns None. + def __reduce__(self, _none_constructor=getattr, _args=(0, "", None)): + return _none_constructor, _args + + else: + + def __reduce__(self, _none_constructor=type(None), _args=()): + return _none_constructor, _args + + +def attrib( + default=NOTHING, + validator=None, + repr=True, + cmp=None, + hash=None, + init=True, + metadata=None, + type=None, + converter=None, + factory=None, + kw_only=False, + eq=None, + order=None, + on_setattr=None, +): + """ + Create a new attribute on a class. + + .. warning:: + + Does *not* do anything unless the class is also decorated with + `attr.s`! + + :param default: A value that is used if an ``attrs``-generated ``__init__`` + is used and no value is passed while instantiating or the attribute is + excluded using ``init=False``. + + If the value is an instance of `Factory`, its callable will be + used to construct a new value (useful for mutable data types like lists + or dicts). + + If a default is not set (or set manually to `attr.NOTHING`), a value + *must* be supplied when instantiating; otherwise a `TypeError` + will be raised. + + The default can also be set using decorator notation as shown below. + + :type default: Any value + + :param callable factory: Syntactic sugar for + ``default=attr.Factory(factory)``. + + :param validator: `callable` that is called by ``attrs``-generated + ``__init__`` methods after the instance has been initialized. They + receive the initialized instance, the `Attribute`, and the + passed value. + + The return value is *not* inspected so the validator has to throw an + exception itself. + + If a `list` is passed, its items are treated as validators and must + all pass. + + Validators can be globally disabled and re-enabled using + `get_run_validators`. + + The validator can also be set using decorator notation as shown below. + + :type validator: `callable` or a `list` of `callable`\\ s. + + :param repr: Include this attribute in the generated ``__repr__`` + method. If ``True``, include the attribute; if ``False``, omit it. By + default, the built-in ``repr()`` function is used. To override how the + attribute value is formatted, pass a ``callable`` that takes a single + value and returns a string. Note that the resulting string is used + as-is, i.e. it will be used directly *instead* of calling ``repr()`` + (the default). + :type repr: a `bool` or a `callable` to use a custom function. + :param bool eq: If ``True`` (default), include this attribute in the + generated ``__eq__`` and ``__ne__`` methods that check two instances + for equality. + :param bool order: If ``True`` (default), include this attributes in the + generated ``__lt__``, ``__le__``, ``__gt__`` and ``__ge__`` methods. + :param bool cmp: Setting to ``True`` is equivalent to setting ``eq=True, + order=True``. Deprecated in favor of *eq* and *order*. + :param Optional[bool] hash: Include this attribute in the generated + ``__hash__`` method. If ``None`` (default), mirror *eq*'s value. This + is the correct behavior according the Python spec. Setting this value + to anything else than ``None`` is *discouraged*. + :param bool init: Include this attribute in the generated ``__init__`` + method. It is possible to set this to ``False`` and set a default + value. In that case this attributed is unconditionally initialized + with the specified default value or factory. + :param callable converter: `callable` that is called by + ``attrs``-generated ``__init__`` methods to convert attribute's value + to the desired format. It is given the passed-in value, and the + returned value will be used as the new value of the attribute. The + value is converted before being passed to the validator, if any. + :param metadata: An arbitrary mapping, to be used by third-party + components. See `extending_metadata`. + :param type: The type of the attribute. In Python 3.6 or greater, the + preferred method to specify the type is using a variable annotation + (see `PEP 526 `_). + This argument is provided for backward compatibility. + Regardless of the approach used, the type will be stored on + ``Attribute.type``. + + Please note that ``attrs`` doesn't do anything with this metadata by + itself. You can use it as part of your own code or for + `static type checking `. + :param kw_only: Make this attribute keyword-only (Python 3+) + in the generated ``__init__`` (if ``init`` is ``False``, this + parameter is ignored). + :param on_setattr: Allows to overwrite the *on_setattr* setting from + `attr.s`. If left `None`, the *on_setattr* value from `attr.s` is used. + Set to `attr.setters.NO_OP` to run **no** `setattr` hooks for this + attribute -- regardless of the setting in `attr.s`. + :type on_setattr: `callable`, or a list of callables, or `None`, or + `attr.setters.NO_OP` + + .. versionadded:: 15.2.0 *convert* + .. versionadded:: 16.3.0 *metadata* + .. versionchanged:: 17.1.0 *validator* can be a ``list`` now. + .. versionchanged:: 17.1.0 + *hash* is ``None`` and therefore mirrors *eq* by default. + .. versionadded:: 17.3.0 *type* + .. deprecated:: 17.4.0 *convert* + .. versionadded:: 17.4.0 *converter* as a replacement for the deprecated + *convert* to achieve consistency with other noun-based arguments. + .. versionadded:: 18.1.0 + ``factory=f`` is syntactic sugar for ``default=attr.Factory(f)``. + .. versionadded:: 18.2.0 *kw_only* + .. versionchanged:: 19.2.0 *convert* keyword argument removed + .. versionchanged:: 19.2.0 *repr* also accepts a custom callable. + .. deprecated:: 19.2.0 *cmp* Removal on or after 2021-06-01. + .. versionadded:: 19.2.0 *eq* and *order* + .. versionadded:: 20.1.0 *on_setattr* + .. versionchanged:: 20.3.0 *kw_only* backported to Python 2 + """ + eq, order = _determine_eq_order(cmp, eq, order, True) + + if hash is not None and hash is not True and hash is not False: + raise TypeError( + "Invalid value for hash. Must be True, False, or None." + ) + + if factory is not None: + if default is not NOTHING: + raise ValueError( + "The `default` and `factory` arguments are mutually " + "exclusive." + ) + if not callable(factory): + raise ValueError("The `factory` argument must be a callable.") + default = Factory(factory) + + if metadata is None: + metadata = {} + + # Apply syntactic sugar by auto-wrapping. + if isinstance(on_setattr, (list, tuple)): + on_setattr = setters.pipe(*on_setattr) + + if validator and isinstance(validator, (list, tuple)): + validator = and_(*validator) + + if converter and isinstance(converter, (list, tuple)): + converter = pipe(*converter) + + return _CountingAttr( + default=default, + validator=validator, + repr=repr, + cmp=None, + hash=hash, + init=init, + converter=converter, + metadata=metadata, + type=type, + kw_only=kw_only, + eq=eq, + order=order, + on_setattr=on_setattr, + ) + + +def _make_attr_tuple_class(cls_name, attr_names): + """ + Create a tuple subclass to hold `Attribute`s for an `attrs` class. + + The subclass is a bare tuple with properties for names. + + class MyClassAttributes(tuple): + __slots__ = () + x = property(itemgetter(0)) + """ + attr_class_name = "{}Attributes".format(cls_name) + attr_class_template = [ + "class {}(tuple):".format(attr_class_name), + " __slots__ = ()", + ] + if attr_names: + for i, attr_name in enumerate(attr_names): + attr_class_template.append( + _tuple_property_pat.format(index=i, attr_name=attr_name) + ) + else: + attr_class_template.append(" pass") + globs = {"_attrs_itemgetter": itemgetter, "_attrs_property": property} + eval(compile("\n".join(attr_class_template), "", "exec"), globs) + + return globs[attr_class_name] + + +# Tuple class for extracted attributes from a class definition. +# `base_attrs` is a subset of `attrs`. +_Attributes = _make_attr_tuple_class( + "_Attributes", + [ + # all attributes to build dunder methods for + "attrs", + # attributes that have been inherited + "base_attrs", + # map inherited attributes to their originating classes + "base_attrs_map", + ], +) + + +def _is_class_var(annot): + """ + Check whether *annot* is a typing.ClassVar. + + The string comparison hack is used to avoid evaluating all string + annotations which would put attrs-based classes at a performance + disadvantage compared to plain old classes. + """ + return str(annot).startswith(_classvar_prefixes) + + +def _has_own_attribute(cls, attrib_name): + """ + Check whether *cls* defines *attrib_name* (and doesn't just inherit it). + + Requires Python 3. + """ + attr = getattr(cls, attrib_name, _sentinel) + if attr is _sentinel: + return False + + for base_cls in cls.__mro__[1:]: + a = getattr(base_cls, attrib_name, None) + if attr is a: + return False + + return True + + +def _get_annotations(cls): + """ + Get annotations for *cls*. + """ + if _has_own_attribute(cls, "__annotations__"): + return cls.__annotations__ + + return {} + + +def _counter_getter(e): + """ + Key function for sorting to avoid re-creating a lambda for every class. + """ + return e[1].counter + + +def _collect_base_attrs(cls, taken_attr_names): + """ + Collect attr.ibs from base classes of *cls*, except *taken_attr_names*. + """ + base_attrs = [] + base_attr_map = {} # A dictionary of base attrs to their classes. + + # Traverse the MRO and collect attributes. + for base_cls in reversed(cls.__mro__[1:-1]): + for a in getattr(base_cls, "__attrs_attrs__", []): + if a.inherited or a.name in taken_attr_names: + continue + + a = a.evolve(inherited=True) + base_attrs.append(a) + base_attr_map[a.name] = base_cls + + # For each name, only keep the freshest definition i.e. the furthest at the + # back. base_attr_map is fine because it gets overwritten with every new + # instance. + filtered = [] + seen = set() + for a in reversed(base_attrs): + if a.name in seen: + continue + filtered.insert(0, a) + seen.add(a.name) + + return filtered, base_attr_map + + +def _collect_base_attrs_broken(cls, taken_attr_names): + """ + Collect attr.ibs from base classes of *cls*, except *taken_attr_names*. + + N.B. *taken_attr_names* will be mutated. + + Adhere to the old incorrect behavior. + + Notably it collects from the front and considers inherited attributes which + leads to the buggy behavior reported in #428. + """ + base_attrs = [] + base_attr_map = {} # A dictionary of base attrs to their classes. + + # Traverse the MRO and collect attributes. + for base_cls in cls.__mro__[1:-1]: + for a in getattr(base_cls, "__attrs_attrs__", []): + if a.name in taken_attr_names: + continue + + a = a.evolve(inherited=True) + taken_attr_names.add(a.name) + base_attrs.append(a) + base_attr_map[a.name] = base_cls + + return base_attrs, base_attr_map + + +def _transform_attrs( + cls, these, auto_attribs, kw_only, collect_by_mro, field_transformer +): + """ + Transform all `_CountingAttr`s on a class into `Attribute`s. + + If *these* is passed, use that and don't look for them on the class. + + *collect_by_mro* is True, collect them in the correct MRO order, otherwise + use the old -- incorrect -- order. See #428. + + Return an `_Attributes`. + """ + cd = cls.__dict__ + anns = _get_annotations(cls) + + if these is not None: + ca_list = [(name, ca) for name, ca in iteritems(these)] + + if not isinstance(these, ordered_dict): + ca_list.sort(key=_counter_getter) + elif auto_attribs is True: + ca_names = { + name + for name, attr in cd.items() + if isinstance(attr, _CountingAttr) + } + ca_list = [] + annot_names = set() + for attr_name, type in anns.items(): + if _is_class_var(type): + continue + annot_names.add(attr_name) + a = cd.get(attr_name, NOTHING) + + if not isinstance(a, _CountingAttr): + if a is NOTHING: + a = attrib() + else: + a = attrib(default=a) + ca_list.append((attr_name, a)) + + unannotated = ca_names - annot_names + if len(unannotated) > 0: + raise UnannotatedAttributeError( + "The following `attr.ib`s lack a type annotation: " + + ", ".join( + sorted(unannotated, key=lambda n: cd.get(n).counter) + ) + + "." + ) + else: + ca_list = sorted( + ( + (name, attr) + for name, attr in cd.items() + if isinstance(attr, _CountingAttr) + ), + key=lambda e: e[1].counter, + ) + + own_attrs = [ + Attribute.from_counting_attr( + name=attr_name, ca=ca, type=anns.get(attr_name) + ) + for attr_name, ca in ca_list + ] + + if collect_by_mro: + base_attrs, base_attr_map = _collect_base_attrs( + cls, {a.name for a in own_attrs} + ) + else: + base_attrs, base_attr_map = _collect_base_attrs_broken( + cls, {a.name for a in own_attrs} + ) + + attr_names = [a.name for a in base_attrs + own_attrs] + + AttrsClass = _make_attr_tuple_class(cls.__name__, attr_names) + + if kw_only: + own_attrs = [a.evolve(kw_only=True) for a in own_attrs] + base_attrs = [a.evolve(kw_only=True) for a in base_attrs] + + attrs = AttrsClass(base_attrs + own_attrs) + + # Mandatory vs non-mandatory attr order only matters when they are part of + # the __init__ signature and when they aren't kw_only (which are moved to + # the end and can be mandatory or non-mandatory in any order, as they will + # be specified as keyword args anyway). Check the order of those attrs: + had_default = False + for a in (a for a in attrs if a.init is not False and a.kw_only is False): + if had_default is True and a.default is NOTHING: + raise ValueError( + "No mandatory attributes allowed after an attribute with a " + "default value or factory. Attribute in question: %r" % (a,) + ) + + if had_default is False and a.default is not NOTHING: + had_default = True + + if field_transformer is not None: + attrs = field_transformer(cls, attrs) + return _Attributes((attrs, base_attrs, base_attr_map)) + + +if PYPY: + + def _frozen_setattrs(self, name, value): + """ + Attached to frozen classes as __setattr__. + """ + if isinstance(self, BaseException) and name in ( + "__cause__", + "__context__", + ): + BaseException.__setattr__(self, name, value) + return + + raise FrozenInstanceError() + + +else: + + def _frozen_setattrs(self, name, value): + """ + Attached to frozen classes as __setattr__. + """ + raise FrozenInstanceError() + + +def _frozen_delattrs(self, name): + """ + Attached to frozen classes as __delattr__. + """ + raise FrozenInstanceError() + + +class _ClassBuilder(object): + """ + Iteratively build *one* class. + """ + + __slots__ = ( + "_attr_names", + "_attrs", + "_base_attr_map", + "_base_names", + "_cache_hash", + "_cls", + "_cls_dict", + "_delete_attribs", + "_frozen", + "_has_post_init", + "_is_exc", + "_on_setattr", + "_slots", + "_weakref_slot", + "_has_own_setattr", + "_has_custom_setattr", + ) + + def __init__( + self, + cls, + these, + slots, + frozen, + weakref_slot, + getstate_setstate, + auto_attribs, + kw_only, + cache_hash, + is_exc, + collect_by_mro, + on_setattr, + has_custom_setattr, + field_transformer, + ): + attrs, base_attrs, base_map = _transform_attrs( + cls, + these, + auto_attribs, + kw_only, + collect_by_mro, + field_transformer, + ) + + self._cls = cls + self._cls_dict = dict(cls.__dict__) if slots else {} + self._attrs = attrs + self._base_names = set(a.name for a in base_attrs) + self._base_attr_map = base_map + self._attr_names = tuple(a.name for a in attrs) + self._slots = slots + self._frozen = frozen + self._weakref_slot = weakref_slot + self._cache_hash = cache_hash + self._has_post_init = bool(getattr(cls, "__attrs_post_init__", False)) + self._delete_attribs = not bool(these) + self._is_exc = is_exc + self._on_setattr = on_setattr + + self._has_custom_setattr = has_custom_setattr + self._has_own_setattr = False + + self._cls_dict["__attrs_attrs__"] = self._attrs + + if frozen: + self._cls_dict["__setattr__"] = _frozen_setattrs + self._cls_dict["__delattr__"] = _frozen_delattrs + + self._has_own_setattr = True + + if getstate_setstate: + ( + self._cls_dict["__getstate__"], + self._cls_dict["__setstate__"], + ) = self._make_getstate_setstate() + + def __repr__(self): + return "<_ClassBuilder(cls={cls})>".format(cls=self._cls.__name__) + + def build_class(self): + """ + Finalize class based on the accumulated configuration. + + Builder cannot be used after calling this method. + """ + if self._slots is True: + return self._create_slots_class() + else: + return self._patch_original_class() + + def _patch_original_class(self): + """ + Apply accumulated methods and return the class. + """ + cls = self._cls + base_names = self._base_names + + # Clean class of attribute definitions (`attr.ib()`s). + if self._delete_attribs: + for name in self._attr_names: + if ( + name not in base_names + and getattr(cls, name, _sentinel) is not _sentinel + ): + try: + delattr(cls, name) + except AttributeError: + # This can happen if a base class defines a class + # variable and we want to set an attribute with the + # same name by using only a type annotation. + pass + + # Attach our dunder methods. + for name, value in self._cls_dict.items(): + setattr(cls, name, value) + + # If we've inherited an attrs __setattr__ and don't write our own, + # reset it to object's. + if not self._has_own_setattr and getattr( + cls, "__attrs_own_setattr__", False + ): + cls.__attrs_own_setattr__ = False + + if not self._has_custom_setattr: + cls.__setattr__ = object.__setattr__ + + return cls + + def _create_slots_class(self): + """ + Build and return a new class with a `__slots__` attribute. + """ + base_names = self._base_names + cd = { + k: v + for k, v in iteritems(self._cls_dict) + if k not in tuple(self._attr_names) + ("__dict__", "__weakref__") + } + + # If our class doesn't have its own implementation of __setattr__ + # (either from the user or by us), check the bases, if one of them has + # an attrs-made __setattr__, that needs to be reset. We don't walk the + # MRO because we only care about our immediate base classes. + # XXX: This can be confused by subclassing a slotted attrs class with + # XXX: a non-attrs class and subclass the resulting class with an attrs + # XXX: class. See `test_slotted_confused` for details. For now that's + # XXX: OK with us. + if not self._has_own_setattr: + cd["__attrs_own_setattr__"] = False + + if not self._has_custom_setattr: + for base_cls in self._cls.__bases__: + if base_cls.__dict__.get("__attrs_own_setattr__", False): + cd["__setattr__"] = object.__setattr__ + break + + # Traverse the MRO to check for an existing __weakref__. + weakref_inherited = False + for base_cls in self._cls.__mro__[1:-1]: + if base_cls.__dict__.get("__weakref__", None) is not None: + weakref_inherited = True + break + + names = self._attr_names + if ( + self._weakref_slot + and "__weakref__" not in getattr(self._cls, "__slots__", ()) + and "__weakref__" not in names + and not weakref_inherited + ): + names += ("__weakref__",) + + # We only add the names of attributes that aren't inherited. + # Setting __slots__ to inherited attributes wastes memory. + slot_names = [name for name in names if name not in base_names] + if self._cache_hash: + slot_names.append(_hash_cache_field) + cd["__slots__"] = tuple(slot_names) + + qualname = getattr(self._cls, "__qualname__", None) + if qualname is not None: + cd["__qualname__"] = qualname + + # Create new class based on old class and our methods. + cls = type(self._cls)(self._cls.__name__, self._cls.__bases__, cd) + + # The following is a fix for + # https://github.com/python-attrs/attrs/issues/102. On Python 3, + # if a method mentions `__class__` or uses the no-arg super(), the + # compiler will bake a reference to the class in the method itself + # as `method.__closure__`. Since we replace the class with a + # clone, we rewrite these references so it keeps working. + for item in cls.__dict__.values(): + if isinstance(item, (classmethod, staticmethod)): + # Class- and staticmethods hide their functions inside. + # These might need to be rewritten as well. + closure_cells = getattr(item.__func__, "__closure__", None) + else: + closure_cells = getattr(item, "__closure__", None) + + if not closure_cells: # Catch None or the empty list. + continue + for cell in closure_cells: + try: + match = cell.cell_contents is self._cls + except ValueError: # ValueError: Cell is empty + pass + else: + if match: + set_closure_cell(cell, cls) + + return cls + + def add_repr(self, ns): + self._cls_dict["__repr__"] = self._add_method_dunders( + _make_repr(self._attrs, ns=ns) + ) + return self + + def add_str(self): + repr = self._cls_dict.get("__repr__") + if repr is None: + raise ValueError( + "__str__ can only be generated if a __repr__ exists." + ) + + def __str__(self): + return self.__repr__() + + self._cls_dict["__str__"] = self._add_method_dunders(__str__) + return self + + def _make_getstate_setstate(self): + """ + Create custom __setstate__ and __getstate__ methods. + """ + # __weakref__ is not writable. + state_attr_names = tuple( + an for an in self._attr_names if an != "__weakref__" + ) + + def slots_getstate(self): + """ + Automatically created by attrs. + """ + return tuple(getattr(self, name) for name in state_attr_names) + + hash_caching_enabled = self._cache_hash + + def slots_setstate(self, state): + """ + Automatically created by attrs. + """ + __bound_setattr = _obj_setattr.__get__(self, Attribute) + for name, value in zip(state_attr_names, state): + __bound_setattr(name, value) + + # The hash code cache is not included when the object is + # serialized, but it still needs to be initialized to None to + # indicate that the first call to __hash__ should be a cache + # miss. + if hash_caching_enabled: + __bound_setattr(_hash_cache_field, None) + + return slots_getstate, slots_setstate + + def make_unhashable(self): + self._cls_dict["__hash__"] = None + return self + + def add_hash(self): + self._cls_dict["__hash__"] = self._add_method_dunders( + _make_hash( + self._cls, + self._attrs, + frozen=self._frozen, + cache_hash=self._cache_hash, + ) + ) + + return self + + def add_init(self): + self._cls_dict["__init__"] = self._add_method_dunders( + _make_init( + self._cls, + self._attrs, + self._has_post_init, + self._frozen, + self._slots, + self._cache_hash, + self._base_attr_map, + self._is_exc, + self._on_setattr is not None + and self._on_setattr is not setters.NO_OP, + ) + ) + + return self + + def add_eq(self): + cd = self._cls_dict + + cd["__eq__"] = self._add_method_dunders( + _make_eq(self._cls, self._attrs) + ) + cd["__ne__"] = self._add_method_dunders(_make_ne()) + + return self + + def add_order(self): + cd = self._cls_dict + + cd["__lt__"], cd["__le__"], cd["__gt__"], cd["__ge__"] = ( + self._add_method_dunders(meth) + for meth in _make_order(self._cls, self._attrs) + ) + + return self + + def add_setattr(self): + if self._frozen: + return self + + sa_attrs = {} + for a in self._attrs: + on_setattr = a.on_setattr or self._on_setattr + if on_setattr and on_setattr is not setters.NO_OP: + sa_attrs[a.name] = a, on_setattr + + if not sa_attrs: + return self + + if self._has_custom_setattr: + # We need to write a __setattr__ but there already is one! + raise ValueError( + "Can't combine custom __setattr__ with on_setattr hooks." + ) + + # docstring comes from _add_method_dunders + def __setattr__(self, name, val): + try: + a, hook = sa_attrs[name] + except KeyError: + nval = val + else: + nval = hook(self, a, val) + + _obj_setattr(self, name, nval) + + self._cls_dict["__attrs_own_setattr__"] = True + self._cls_dict["__setattr__"] = self._add_method_dunders(__setattr__) + self._has_own_setattr = True + + return self + + def _add_method_dunders(self, method): + """ + Add __module__ and __qualname__ to a *method* if possible. + """ + try: + method.__module__ = self._cls.__module__ + except AttributeError: + pass + + try: + method.__qualname__ = ".".join( + (self._cls.__qualname__, method.__name__) + ) + except AttributeError: + pass + + try: + method.__doc__ = "Method generated by attrs for class %s." % ( + self._cls.__qualname__, + ) + except AttributeError: + pass + + return method + + +_CMP_DEPRECATION = ( + "The usage of `cmp` is deprecated and will be removed on or after " + "2021-06-01. Please use `eq` and `order` instead." +) + + +def _determine_eq_order(cmp, eq, order, default_eq): + """ + Validate the combination of *cmp*, *eq*, and *order*. Derive the effective + values of eq and order. If *eq* is None, set it to *default_eq*. + """ + if cmp is not None and any((eq is not None, order is not None)): + raise ValueError("Don't mix `cmp` with `eq' and `order`.") + + # cmp takes precedence due to bw-compatibility. + if cmp is not None: + warnings.warn(_CMP_DEPRECATION, DeprecationWarning, stacklevel=3) + + return cmp, cmp + + # If left None, equality is set to the specified default and ordering + # mirrors equality. + if eq is None: + eq = default_eq + + if order is None: + order = eq + + if eq is False and order is True: + raise ValueError("`order` can only be True if `eq` is True too.") + + return eq, order + + +def _determine_whether_to_implement( + cls, flag, auto_detect, dunders, default=True +): + """ + Check whether we should implement a set of methods for *cls*. + + *flag* is the argument passed into @attr.s like 'init', *auto_detect* the + same as passed into @attr.s and *dunders* is a tuple of attribute names + whose presence signal that the user has implemented it themselves. + + Return *default* if no reason for either for or against is found. + + auto_detect must be False on Python 2. + """ + if flag is True or flag is False: + return flag + + if flag is None and auto_detect is False: + return default + + # Logically, flag is None and auto_detect is True here. + for dunder in dunders: + if _has_own_attribute(cls, dunder): + return False + + return default + + +def attrs( + maybe_cls=None, + these=None, + repr_ns=None, + repr=None, + cmp=None, + hash=None, + init=None, + slots=False, + frozen=False, + weakref_slot=True, + str=False, + auto_attribs=False, + kw_only=False, + cache_hash=False, + auto_exc=False, + eq=None, + order=None, + auto_detect=False, + collect_by_mro=False, + getstate_setstate=None, + on_setattr=None, + field_transformer=None, +): + r""" + A class decorator that adds `dunder + `_\ -methods according to the + specified attributes using `attr.ib` or the *these* argument. + + :param these: A dictionary of name to `attr.ib` mappings. This is + useful to avoid the definition of your attributes within the class body + because you can't (e.g. if you want to add ``__repr__`` methods to + Django models) or don't want to. + + If *these* is not ``None``, ``attrs`` will *not* search the class body + for attributes and will *not* remove any attributes from it. + + If *these* is an ordered dict (`dict` on Python 3.6+, + `collections.OrderedDict` otherwise), the order is deduced from + the order of the attributes inside *these*. Otherwise the order + of the definition of the attributes is used. + + :type these: `dict` of `str` to `attr.ib` + + :param str repr_ns: When using nested classes, there's no way in Python 2 + to automatically detect that. Therefore it's possible to set the + namespace explicitly for a more meaningful ``repr`` output. + :param bool auto_detect: Instead of setting the *init*, *repr*, *eq*, + *order*, and *hash* arguments explicitly, assume they are set to + ``True`` **unless any** of the involved methods for one of the + arguments is implemented in the *current* class (i.e. it is *not* + inherited from some base class). + + So for example by implementing ``__eq__`` on a class yourself, + ``attrs`` will deduce ``eq=False`` and won't create *neither* + ``__eq__`` *nor* ``__ne__`` (but Python classes come with a sensible + ``__ne__`` by default, so it *should* be enough to only implement + ``__eq__`` in most cases). + + .. warning:: + + If you prevent ``attrs`` from creating the ordering methods for you + (``order=False``, e.g. by implementing ``__le__``), it becomes + *your* responsibility to make sure its ordering is sound. The best + way is to use the `functools.total_ordering` decorator. + + + Passing ``True`` or ``False`` to *init*, *repr*, *eq*, *order*, + *cmp*, or *hash* overrides whatever *auto_detect* would determine. + + *auto_detect* requires Python 3. Setting it ``True`` on Python 2 raises + a `PythonTooOldError`. + + :param bool repr: Create a ``__repr__`` method with a human readable + representation of ``attrs`` attributes.. + :param bool str: Create a ``__str__`` method that is identical to + ``__repr__``. This is usually not necessary except for + `Exception`\ s. + :param Optional[bool] eq: If ``True`` or ``None`` (default), add ``__eq__`` + and ``__ne__`` methods that check two instances for equality. + + They compare the instances as if they were tuples of their ``attrs`` + attributes if and only if the types of both classes are *identical*! + :param Optional[bool] order: If ``True``, add ``__lt__``, ``__le__``, + ``__gt__``, and ``__ge__`` methods that behave like *eq* above and + allow instances to be ordered. If ``None`` (default) mirror value of + *eq*. + :param Optional[bool] cmp: Setting to ``True`` is equivalent to setting + ``eq=True, order=True``. Deprecated in favor of *eq* and *order*, has + precedence over them for backward-compatibility though. Must not be + mixed with *eq* or *order*. + :param Optional[bool] hash: If ``None`` (default), the ``__hash__`` method + is generated according how *eq* and *frozen* are set. + + 1. If *both* are True, ``attrs`` will generate a ``__hash__`` for you. + 2. If *eq* is True and *frozen* is False, ``__hash__`` will be set to + None, marking it unhashable (which it is). + 3. If *eq* is False, ``__hash__`` will be left untouched meaning the + ``__hash__`` method of the base class will be used (if base class is + ``object``, this means it will fall back to id-based hashing.). + + Although not recommended, you can decide for yourself and force + ``attrs`` to create one (e.g. if the class is immutable even though you + didn't freeze it programmatically) by passing ``True`` or not. Both of + these cases are rather special and should be used carefully. + + See our documentation on `hashing`, Python's documentation on + `object.__hash__`, and the `GitHub issue that led to the default \ + behavior `_ for more + details. + :param bool init: Create a ``__init__`` method that initializes the + ``attrs`` attributes. Leading underscores are stripped for the + argument name. If a ``__attrs_post_init__`` method exists on the + class, it will be called after the class is fully initialized. + :param bool slots: Create a `slotted class ` that's more + memory-efficient. Slotted classes are generally superior to the default + dict classes, but have some gotchas you should know about, so we + encourage you to read the `glossary entry `. + :param bool frozen: Make instances immutable after initialization. If + someone attempts to modify a frozen instance, + `attr.exceptions.FrozenInstanceError` is raised. + + .. note:: + + 1. This is achieved by installing a custom ``__setattr__`` method + on your class, so you can't implement your own. + + 2. True immutability is impossible in Python. + + 3. This *does* have a minor a runtime performance `impact + ` when initializing new instances. In other words: + ``__init__`` is slightly slower with ``frozen=True``. + + 4. If a class is frozen, you cannot modify ``self`` in + ``__attrs_post_init__`` or a self-written ``__init__``. You can + circumvent that limitation by using + ``object.__setattr__(self, "attribute_name", value)``. + + 5. Subclasses of a frozen class are frozen too. + + :param bool weakref_slot: Make instances weak-referenceable. This has no + effect unless ``slots`` is also enabled. + :param bool auto_attribs: If ``True``, collect `PEP 526`_-annotated + attributes (Python 3.6 and later only) from the class body. + + In this case, you **must** annotate every field. If ``attrs`` + encounters a field that is set to an `attr.ib` but lacks a type + annotation, an `attr.exceptions.UnannotatedAttributeError` is + raised. Use ``field_name: typing.Any = attr.ib(...)`` if you don't + want to set a type. + + If you assign a value to those attributes (e.g. ``x: int = 42``), that + value becomes the default value like if it were passed using + ``attr.ib(default=42)``. Passing an instance of `Factory` also + works as expected. + + Attributes annotated as `typing.ClassVar`, and attributes that are + neither annotated nor set to an `attr.ib` are **ignored**. + + .. _`PEP 526`: https://www.python.org/dev/peps/pep-0526/ + :param bool kw_only: Make all attributes keyword-only (Python 3+) + in the generated ``__init__`` (if ``init`` is ``False``, this + parameter is ignored). + :param bool cache_hash: Ensure that the object's hash code is computed + only once and stored on the object. If this is set to ``True``, + hashing must be either explicitly or implicitly enabled for this + class. If the hash code is cached, avoid any reassignments of + fields involved in hash code computation or mutations of the objects + those fields point to after object creation. If such changes occur, + the behavior of the object's hash code is undefined. + :param bool auto_exc: If the class subclasses `BaseException` + (which implicitly includes any subclass of any exception), the + following happens to behave like a well-behaved Python exceptions + class: + + - the values for *eq*, *order*, and *hash* are ignored and the + instances compare and hash by the instance's ids (N.B. ``attrs`` will + *not* remove existing implementations of ``__hash__`` or the equality + methods. It just won't add own ones.), + - all attributes that are either passed into ``__init__`` or have a + default value are additionally available as a tuple in the ``args`` + attribute, + - the value of *str* is ignored leaving ``__str__`` to base classes. + :param bool collect_by_mro: Setting this to `True` fixes the way ``attrs`` + collects attributes from base classes. The default behavior is + incorrect in certain cases of multiple inheritance. It should be on by + default but is kept off for backward-compatability. + + See issue `#428 `_ for + more details. + + :param Optional[bool] getstate_setstate: + .. note:: + This is usually only interesting for slotted classes and you should + probably just set *auto_detect* to `True`. + + If `True`, ``__getstate__`` and + ``__setstate__`` are generated and attached to the class. This is + necessary for slotted classes to be pickleable. If left `None`, it's + `True` by default for slotted classes and ``False`` for dict classes. + + If *auto_detect* is `True`, and *getstate_setstate* is left `None`, + and **either** ``__getstate__`` or ``__setstate__`` is detected directly + on the class (i.e. not inherited), it is set to `False` (this is usually + what you want). + + :param on_setattr: A callable that is run whenever the user attempts to set + an attribute (either by assignment like ``i.x = 42`` or by using + `setattr` like ``setattr(i, "x", 42)``). It receives the same arguments + as validators: the instance, the attribute that is being modified, and + the new value. + + If no exception is raised, the attribute is set to the return value of + the callable. + + If a list of callables is passed, they're automatically wrapped in an + `attr.setters.pipe`. + + :param Optional[callable] field_transformer: + A function that is called with the original class object and all + fields right before ``attrs`` finalizes the class. You can use + this, e.g., to automatically add converters or validators to + fields based on their types. See `transform-fields` for more details. + + .. versionadded:: 16.0.0 *slots* + .. versionadded:: 16.1.0 *frozen* + .. versionadded:: 16.3.0 *str* + .. versionadded:: 16.3.0 Support for ``__attrs_post_init__``. + .. versionchanged:: 17.1.0 + *hash* supports ``None`` as value which is also the default now. + .. versionadded:: 17.3.0 *auto_attribs* + .. versionchanged:: 18.1.0 + If *these* is passed, no attributes are deleted from the class body. + .. versionchanged:: 18.1.0 If *these* is ordered, the order is retained. + .. versionadded:: 18.2.0 *weakref_slot* + .. deprecated:: 18.2.0 + ``__lt__``, ``__le__``, ``__gt__``, and ``__ge__`` now raise a + `DeprecationWarning` if the classes compared are subclasses of + each other. ``__eq`` and ``__ne__`` never tried to compared subclasses + to each other. + .. versionchanged:: 19.2.0 + ``__lt__``, ``__le__``, ``__gt__``, and ``__ge__`` now do not consider + subclasses comparable anymore. + .. versionadded:: 18.2.0 *kw_only* + .. versionadded:: 18.2.0 *cache_hash* + .. versionadded:: 19.1.0 *auto_exc* + .. deprecated:: 19.2.0 *cmp* Removal on or after 2021-06-01. + .. versionadded:: 19.2.0 *eq* and *order* + .. versionadded:: 20.1.0 *auto_detect* + .. versionadded:: 20.1.0 *collect_by_mro* + .. versionadded:: 20.1.0 *getstate_setstate* + .. versionadded:: 20.1.0 *on_setattr* + .. versionadded:: 20.3.0 *field_transformer* + """ + if auto_detect and PY2: + raise PythonTooOldError( + "auto_detect only works on Python 3 and later." + ) + + eq_, order_ = _determine_eq_order(cmp, eq, order, None) + hash_ = hash # work around the lack of nonlocal + + if isinstance(on_setattr, (list, tuple)): + on_setattr = setters.pipe(*on_setattr) + + def wrap(cls): + + if getattr(cls, "__class__", None) is None: + raise TypeError("attrs only works with new-style classes.") + + is_frozen = frozen or _has_frozen_base_class(cls) + is_exc = auto_exc is True and issubclass(cls, BaseException) + has_own_setattr = auto_detect and _has_own_attribute( + cls, "__setattr__" + ) + + if has_own_setattr and is_frozen: + raise ValueError("Can't freeze a class with a custom __setattr__.") + + builder = _ClassBuilder( + cls, + these, + slots, + is_frozen, + weakref_slot, + _determine_whether_to_implement( + cls, + getstate_setstate, + auto_detect, + ("__getstate__", "__setstate__"), + default=slots, + ), + auto_attribs, + kw_only, + cache_hash, + is_exc, + collect_by_mro, + on_setattr, + has_own_setattr, + field_transformer, + ) + if _determine_whether_to_implement( + cls, repr, auto_detect, ("__repr__",) + ): + builder.add_repr(repr_ns) + if str is True: + builder.add_str() + + eq = _determine_whether_to_implement( + cls, eq_, auto_detect, ("__eq__", "__ne__") + ) + if not is_exc and eq is True: + builder.add_eq() + if not is_exc and _determine_whether_to_implement( + cls, order_, auto_detect, ("__lt__", "__le__", "__gt__", "__ge__") + ): + builder.add_order() + + builder.add_setattr() + + if ( + hash_ is None + and auto_detect is True + and _has_own_attribute(cls, "__hash__") + ): + hash = False + else: + hash = hash_ + if hash is not True and hash is not False and hash is not None: + # Can't use `hash in` because 1 == True for example. + raise TypeError( + "Invalid value for hash. Must be True, False, or None." + ) + elif hash is False or (hash is None and eq is False) or is_exc: + # Don't do anything. Should fall back to __object__'s __hash__ + # which is by id. + if cache_hash: + raise TypeError( + "Invalid value for cache_hash. To use hash caching," + " hashing must be either explicitly or implicitly " + "enabled." + ) + elif hash is True or ( + hash is None and eq is True and is_frozen is True + ): + # Build a __hash__ if told so, or if it's safe. + builder.add_hash() + else: + # Raise TypeError on attempts to hash. + if cache_hash: + raise TypeError( + "Invalid value for cache_hash. To use hash caching," + " hashing must be either explicitly or implicitly " + "enabled." + ) + builder.make_unhashable() + + if _determine_whether_to_implement( + cls, init, auto_detect, ("__init__",) + ): + builder.add_init() + else: + if cache_hash: + raise TypeError( + "Invalid value for cache_hash. To use hash caching," + " init must be True." + ) + + return builder.build_class() + + # maybe_cls's type depends on the usage of the decorator. It's a class + # if it's used as `@attrs` but ``None`` if used as `@attrs()`. + if maybe_cls is None: + return wrap + else: + return wrap(maybe_cls) + + +_attrs = attrs +""" +Internal alias so we can use it in functions that take an argument called +*attrs*. +""" + + +if PY2: + + def _has_frozen_base_class(cls): + """ + Check whether *cls* has a frozen ancestor by looking at its + __setattr__. + """ + return ( + getattr(cls.__setattr__, "__module__", None) + == _frozen_setattrs.__module__ + and cls.__setattr__.__name__ == _frozen_setattrs.__name__ + ) + + +else: + + def _has_frozen_base_class(cls): + """ + Check whether *cls* has a frozen ancestor by looking at its + __setattr__. + """ + return cls.__setattr__ == _frozen_setattrs + + +def _attrs_to_tuple(obj, attrs): + """ + Create a tuple of all values of *obj*'s *attrs*. + """ + return tuple(getattr(obj, a.name) for a in attrs) + + +def _generate_unique_filename(cls, func_name): + """ + Create a "filename" suitable for a function being generated. + """ + unique_id = uuid.uuid4() + extra = "" + count = 1 + + while True: + unique_filename = "".format( + func_name, + cls.__module__, + getattr(cls, "__qualname__", cls.__name__), + extra, + ) + # To handle concurrency we essentially "reserve" our spot in + # the linecache with a dummy line. The caller can then + # set this value correctly. + cache_line = (1, None, (str(unique_id),), unique_filename) + if ( + linecache.cache.setdefault(unique_filename, cache_line) + == cache_line + ): + return unique_filename + + # Looks like this spot is taken. Try again. + count += 1 + extra = "-{0}".format(count) + + +def _make_hash(cls, attrs, frozen, cache_hash): + attrs = tuple( + a for a in attrs if a.hash is True or (a.hash is None and a.eq is True) + ) + + tab = " " + + unique_filename = _generate_unique_filename(cls, "hash") + type_hash = hash(unique_filename) + + hash_def = "def __hash__(self" + hash_func = "hash((" + closing_braces = "))" + if not cache_hash: + hash_def += "):" + else: + if not PY2: + hash_def += ", *" + + hash_def += ( + ", _cache_wrapper=" + + "__import__('attr._make')._make._CacheHashWrapper):" + ) + hash_func = "_cache_wrapper(" + hash_func + closing_braces += ")" + + method_lines = [hash_def] + + def append_hash_computation_lines(prefix, indent): + """ + Generate the code for actually computing the hash code. + Below this will either be returned directly or used to compute + a value which is then cached, depending on the value of cache_hash + """ + + method_lines.extend( + [ + indent + prefix + hash_func, + indent + " %d," % (type_hash,), + ] + ) + + for a in attrs: + method_lines.append(indent + " self.%s," % a.name) + + method_lines.append(indent + " " + closing_braces) + + if cache_hash: + method_lines.append(tab + "if self.%s is None:" % _hash_cache_field) + if frozen: + append_hash_computation_lines( + "object.__setattr__(self, '%s', " % _hash_cache_field, tab * 2 + ) + method_lines.append(tab * 2 + ")") # close __setattr__ + else: + append_hash_computation_lines( + "self.%s = " % _hash_cache_field, tab * 2 + ) + method_lines.append(tab + "return self.%s" % _hash_cache_field) + else: + append_hash_computation_lines("return ", tab) + + script = "\n".join(method_lines) + globs = {} + locs = {} + bytecode = compile(script, unique_filename, "exec") + eval(bytecode, globs, locs) + + # In order of debuggers like PDB being able to step through the code, + # we add a fake linecache entry. + linecache.cache[unique_filename] = ( + len(script), + None, + script.splitlines(True), + unique_filename, + ) + + return locs["__hash__"] + + +def _add_hash(cls, attrs): + """ + Add a hash method to *cls*. + """ + cls.__hash__ = _make_hash(cls, attrs, frozen=False, cache_hash=False) + return cls + + +def _make_ne(): + """ + Create __ne__ method. + """ + + def __ne__(self, other): + """ + Check equality and either forward a NotImplemented or + return the result negated. + """ + result = self.__eq__(other) + if result is NotImplemented: + return NotImplemented + + return not result + + return __ne__ + + +def _make_eq(cls, attrs): + """ + Create __eq__ method for *cls* with *attrs*. + """ + attrs = [a for a in attrs if a.eq] + + unique_filename = _generate_unique_filename(cls, "eq") + lines = [ + "def __eq__(self, other):", + " if other.__class__ is not self.__class__:", + " return NotImplemented", + ] + # We can't just do a big self.x = other.x and... clause due to + # irregularities like nan == nan is false but (nan,) == (nan,) is true. + if attrs: + lines.append(" return (") + others = [" ) == ("] + for a in attrs: + lines.append(" self.%s," % (a.name,)) + others.append(" other.%s," % (a.name,)) + + lines += others + [" )"] + else: + lines.append(" return True") + + script = "\n".join(lines) + globs = {} + locs = {} + bytecode = compile(script, unique_filename, "exec") + eval(bytecode, globs, locs) + + # In order of debuggers like PDB being able to step through the code, + # we add a fake linecache entry. + linecache.cache[unique_filename] = ( + len(script), + None, + script.splitlines(True), + unique_filename, + ) + return locs["__eq__"] + + +def _make_order(cls, attrs): + """ + Create ordering methods for *cls* with *attrs*. + """ + attrs = [a for a in attrs if a.order] + + def attrs_to_tuple(obj): + """ + Save us some typing. + """ + return _attrs_to_tuple(obj, attrs) + + def __lt__(self, other): + """ + Automatically created by attrs. + """ + if other.__class__ is self.__class__: + return attrs_to_tuple(self) < attrs_to_tuple(other) + + return NotImplemented + + def __le__(self, other): + """ + Automatically created by attrs. + """ + if other.__class__ is self.__class__: + return attrs_to_tuple(self) <= attrs_to_tuple(other) + + return NotImplemented + + def __gt__(self, other): + """ + Automatically created by attrs. + """ + if other.__class__ is self.__class__: + return attrs_to_tuple(self) > attrs_to_tuple(other) + + return NotImplemented + + def __ge__(self, other): + """ + Automatically created by attrs. + """ + if other.__class__ is self.__class__: + return attrs_to_tuple(self) >= attrs_to_tuple(other) + + return NotImplemented + + return __lt__, __le__, __gt__, __ge__ + + +def _add_eq(cls, attrs=None): + """ + Add equality methods to *cls* with *attrs*. + """ + if attrs is None: + attrs = cls.__attrs_attrs__ + + cls.__eq__ = _make_eq(cls, attrs) + cls.__ne__ = _make_ne() + + return cls + + +_already_repring = threading.local() + + +def _make_repr(attrs, ns): + """ + Make a repr method that includes relevant *attrs*, adding *ns* to the full + name. + """ + + # Figure out which attributes to include, and which function to use to + # format them. The a.repr value can be either bool or a custom callable. + attr_names_with_reprs = tuple( + (a.name, repr if a.repr is True else a.repr) + for a in attrs + if a.repr is not False + ) + + def __repr__(self): + """ + Automatically created by attrs. + """ + try: + working_set = _already_repring.working_set + except AttributeError: + working_set = set() + _already_repring.working_set = working_set + + if id(self) in working_set: + return "..." + real_cls = self.__class__ + if ns is None: + qualname = getattr(real_cls, "__qualname__", None) + if qualname is not None: + class_name = qualname.rsplit(">.", 1)[-1] + else: + class_name = real_cls.__name__ + else: + class_name = ns + "." + real_cls.__name__ + + # Since 'self' remains on the stack (i.e.: strongly referenced) for the + # duration of this call, it's safe to depend on id(...) stability, and + # not need to track the instance and therefore worry about properties + # like weakref- or hash-ability. + working_set.add(id(self)) + try: + result = [class_name, "("] + first = True + for name, attr_repr in attr_names_with_reprs: + if first: + first = False + else: + result.append(", ") + result.extend( + (name, "=", attr_repr(getattr(self, name, NOTHING))) + ) + return "".join(result) + ")" + finally: + working_set.remove(id(self)) + + return __repr__ + + +def _add_repr(cls, ns=None, attrs=None): + """ + Add a repr method to *cls*. + """ + if attrs is None: + attrs = cls.__attrs_attrs__ + + cls.__repr__ = _make_repr(attrs, ns) + return cls + + +def fields(cls): + """ + Return the tuple of ``attrs`` attributes for a class. + + The tuple also allows accessing the fields by their names (see below for + examples). + + :param type cls: Class to introspect. + + :raise TypeError: If *cls* is not a class. + :raise attr.exceptions.NotAnAttrsClassError: If *cls* is not an ``attrs`` + class. + + :rtype: tuple (with name accessors) of `attr.Attribute` + + .. versionchanged:: 16.2.0 Returned tuple allows accessing the fields + by name. + """ + if not isclass(cls): + raise TypeError("Passed object must be a class.") + attrs = getattr(cls, "__attrs_attrs__", None) + if attrs is None: + raise NotAnAttrsClassError( + "{cls!r} is not an attrs-decorated class.".format(cls=cls) + ) + return attrs + + +def fields_dict(cls): + """ + Return an ordered dictionary of ``attrs`` attributes for a class, whose + keys are the attribute names. + + :param type cls: Class to introspect. + + :raise TypeError: If *cls* is not a class. + :raise attr.exceptions.NotAnAttrsClassError: If *cls* is not an ``attrs`` + class. + + :rtype: an ordered dict where keys are attribute names and values are + `attr.Attribute`\\ s. This will be a `dict` if it's + naturally ordered like on Python 3.6+ or an + :class:`~collections.OrderedDict` otherwise. + + .. versionadded:: 18.1.0 + """ + if not isclass(cls): + raise TypeError("Passed object must be a class.") + attrs = getattr(cls, "__attrs_attrs__", None) + if attrs is None: + raise NotAnAttrsClassError( + "{cls!r} is not an attrs-decorated class.".format(cls=cls) + ) + return ordered_dict(((a.name, a) for a in attrs)) + + +def validate(inst): + """ + Validate all attributes on *inst* that have a validator. + + Leaves all exceptions through. + + :param inst: Instance of a class with ``attrs`` attributes. + """ + if _config._run_validators is False: + return + + for a in fields(inst.__class__): + v = a.validator + if v is not None: + v(inst, a, getattr(inst, a.name)) + + +def _is_slot_cls(cls): + return "__slots__" in cls.__dict__ + + +def _is_slot_attr(a_name, base_attr_map): + """ + Check if the attribute name comes from a slot class. + """ + return a_name in base_attr_map and _is_slot_cls(base_attr_map[a_name]) + + +def _make_init( + cls, + attrs, + post_init, + frozen, + slots, + cache_hash, + base_attr_map, + is_exc, + has_global_on_setattr, +): + if frozen and has_global_on_setattr: + raise ValueError("Frozen classes can't use on_setattr.") + + needs_cached_setattr = cache_hash or frozen + filtered_attrs = [] + attr_dict = {} + for a in attrs: + if not a.init and a.default is NOTHING: + continue + + filtered_attrs.append(a) + attr_dict[a.name] = a + + if a.on_setattr is not None: + if frozen is True: + raise ValueError("Frozen classes can't use on_setattr.") + + needs_cached_setattr = True + elif ( + has_global_on_setattr and a.on_setattr is not setters.NO_OP + ) or _is_slot_attr(a.name, base_attr_map): + needs_cached_setattr = True + + unique_filename = _generate_unique_filename(cls, "init") + + script, globs, annotations = _attrs_to_init_script( + filtered_attrs, + frozen, + slots, + post_init, + cache_hash, + base_attr_map, + is_exc, + needs_cached_setattr, + has_global_on_setattr, + ) + locs = {} + bytecode = compile(script, unique_filename, "exec") + globs.update({"NOTHING": NOTHING, "attr_dict": attr_dict}) + + if needs_cached_setattr: + # Save the lookup overhead in __init__ if we need to circumvent + # setattr hooks. + globs["_cached_setattr"] = _obj_setattr + + eval(bytecode, globs, locs) + + # In order of debuggers like PDB being able to step through the code, + # we add a fake linecache entry. + linecache.cache[unique_filename] = ( + len(script), + None, + script.splitlines(True), + unique_filename, + ) + + __init__ = locs["__init__"] + __init__.__annotations__ = annotations + + return __init__ + + +def _setattr(attr_name, value_var, has_on_setattr): + """ + Use the cached object.setattr to set *attr_name* to *value_var*. + """ + return "_setattr('%s', %s)" % (attr_name, value_var) + + +def _setattr_with_converter(attr_name, value_var, has_on_setattr): + """ + Use the cached object.setattr to set *attr_name* to *value_var*, but run + its converter first. + """ + return "_setattr('%s', %s(%s))" % ( + attr_name, + _init_converter_pat % (attr_name,), + value_var, + ) + + +def _assign(attr_name, value, has_on_setattr): + """ + Unless *attr_name* has an on_setattr hook, use normal assignment. Otherwise + relegate to _setattr. + """ + if has_on_setattr: + return _setattr(attr_name, value, True) + + return "self.%s = %s" % (attr_name, value) + + +def _assign_with_converter(attr_name, value_var, has_on_setattr): + """ + Unless *attr_name* has an on_setattr hook, use normal assignment after + conversion. Otherwise relegate to _setattr_with_converter. + """ + if has_on_setattr: + return _setattr_with_converter(attr_name, value_var, True) + + return "self.%s = %s(%s)" % ( + attr_name, + _init_converter_pat % (attr_name,), + value_var, + ) + + +if PY2: + + def _unpack_kw_only_py2(attr_name, default=None): + """ + Unpack *attr_name* from _kw_only dict. + """ + if default is not None: + arg_default = ", %s" % default + else: + arg_default = "" + return "%s = _kw_only.pop('%s'%s)" % ( + attr_name, + attr_name, + arg_default, + ) + + def _unpack_kw_only_lines_py2(kw_only_args): + """ + Unpack all *kw_only_args* from _kw_only dict and handle errors. + + Given a list of strings "{attr_name}" and "{attr_name}={default}" + generates list of lines of code that pop attrs from _kw_only dict and + raise TypeError similar to builtin if required attr is missing or + extra key is passed. + + >>> print("\n".join(_unpack_kw_only_lines_py2(["a", "b=42"]))) + try: + a = _kw_only.pop('a') + b = _kw_only.pop('b', 42) + except KeyError as _key_error: + raise TypeError( + ... + if _kw_only: + raise TypeError( + ... + """ + lines = ["try:"] + lines.extend( + " " + _unpack_kw_only_py2(*arg.split("=")) + for arg in kw_only_args + ) + lines += """\ +except KeyError as _key_error: + raise TypeError( + '__init__() missing required keyword-only argument: %s' % _key_error + ) +if _kw_only: + raise TypeError( + '__init__() got an unexpected keyword argument %r' + % next(iter(_kw_only)) + ) +""".split( + "\n" + ) + return lines + + +def _attrs_to_init_script( + attrs, + frozen, + slots, + post_init, + cache_hash, + base_attr_map, + is_exc, + needs_cached_setattr, + has_global_on_setattr, +): + """ + Return a script of an initializer for *attrs* and a dict of globals. + + The globals are expected by the generated script. + + If *frozen* is True, we cannot set the attributes directly so we use + a cached ``object.__setattr__``. + """ + lines = [] + if needs_cached_setattr: + lines.append( + # Circumvent the __setattr__ descriptor to save one lookup per + # assignment. + # Note _setattr will be used again below if cache_hash is True + "_setattr = _cached_setattr.__get__(self, self.__class__)" + ) + + if frozen is True: + if slots is True: + fmt_setter = _setattr + fmt_setter_with_converter = _setattr_with_converter + else: + # Dict frozen classes assign directly to __dict__. + # But only if the attribute doesn't come from an ancestor slot + # class. + # Note _inst_dict will be used again below if cache_hash is True + lines.append("_inst_dict = self.__dict__") + + def fmt_setter(attr_name, value_var, has_on_setattr): + if _is_slot_attr(attr_name, base_attr_map): + return _setattr(attr_name, value_var, has_on_setattr) + + return "_inst_dict['%s'] = %s" % (attr_name, value_var) + + def fmt_setter_with_converter( + attr_name, value_var, has_on_setattr + ): + if has_on_setattr or _is_slot_attr(attr_name, base_attr_map): + return _setattr_with_converter( + attr_name, value_var, has_on_setattr + ) + + return "_inst_dict['%s'] = %s(%s)" % ( + attr_name, + _init_converter_pat % (attr_name,), + value_var, + ) + + else: + # Not frozen. + fmt_setter = _assign + fmt_setter_with_converter = _assign_with_converter + + args = [] + kw_only_args = [] + attrs_to_validate = [] + + # This is a dictionary of names to validator and converter callables. + # Injecting this into __init__ globals lets us avoid lookups. + names_for_globals = {} + annotations = {"return": None} + + for a in attrs: + if a.validator: + attrs_to_validate.append(a) + + attr_name = a.name + has_on_setattr = a.on_setattr is not None or ( + a.on_setattr is not setters.NO_OP and has_global_on_setattr + ) + arg_name = a.name.lstrip("_") + + has_factory = isinstance(a.default, Factory) + if has_factory and a.default.takes_self: + maybe_self = "self" + else: + maybe_self = "" + + if a.init is False: + if has_factory: + init_factory_name = _init_factory_pat.format(a.name) + if a.converter is not None: + lines.append( + fmt_setter_with_converter( + attr_name, + init_factory_name + "(%s)" % (maybe_self,), + has_on_setattr, + ) + ) + conv_name = _init_converter_pat % (a.name,) + names_for_globals[conv_name] = a.converter + else: + lines.append( + fmt_setter( + attr_name, + init_factory_name + "(%s)" % (maybe_self,), + has_on_setattr, + ) + ) + names_for_globals[init_factory_name] = a.default.factory + else: + if a.converter is not None: + lines.append( + fmt_setter_with_converter( + attr_name, + "attr_dict['%s'].default" % (attr_name,), + has_on_setattr, + ) + ) + conv_name = _init_converter_pat % (a.name,) + names_for_globals[conv_name] = a.converter + else: + lines.append( + fmt_setter( + attr_name, + "attr_dict['%s'].default" % (attr_name,), + has_on_setattr, + ) + ) + elif a.default is not NOTHING and not has_factory: + arg = "%s=attr_dict['%s'].default" % (arg_name, attr_name) + if a.kw_only: + kw_only_args.append(arg) + else: + args.append(arg) + + if a.converter is not None: + lines.append( + fmt_setter_with_converter( + attr_name, arg_name, has_on_setattr + ) + ) + names_for_globals[ + _init_converter_pat % (a.name,) + ] = a.converter + else: + lines.append(fmt_setter(attr_name, arg_name, has_on_setattr)) + + elif has_factory: + arg = "%s=NOTHING" % (arg_name,) + if a.kw_only: + kw_only_args.append(arg) + else: + args.append(arg) + lines.append("if %s is not NOTHING:" % (arg_name,)) + + init_factory_name = _init_factory_pat.format(a.name) + if a.converter is not None: + lines.append( + " " + + fmt_setter_with_converter( + attr_name, arg_name, has_on_setattr + ) + ) + lines.append("else:") + lines.append( + " " + + fmt_setter_with_converter( + attr_name, + init_factory_name + "(" + maybe_self + ")", + has_on_setattr, + ) + ) + names_for_globals[ + _init_converter_pat % (a.name,) + ] = a.converter + else: + lines.append( + " " + fmt_setter(attr_name, arg_name, has_on_setattr) + ) + lines.append("else:") + lines.append( + " " + + fmt_setter( + attr_name, + init_factory_name + "(" + maybe_self + ")", + has_on_setattr, + ) + ) + names_for_globals[init_factory_name] = a.default.factory + else: + if a.kw_only: + kw_only_args.append(arg_name) + else: + args.append(arg_name) + + if a.converter is not None: + lines.append( + fmt_setter_with_converter( + attr_name, arg_name, has_on_setattr + ) + ) + names_for_globals[ + _init_converter_pat % (a.name,) + ] = a.converter + else: + lines.append(fmt_setter(attr_name, arg_name, has_on_setattr)) + + if a.init is True and a.converter is None and a.type is not None: + annotations[arg_name] = a.type + + if attrs_to_validate: # we can skip this if there are no validators. + names_for_globals["_config"] = _config + lines.append("if _config._run_validators is True:") + for a in attrs_to_validate: + val_name = "__attr_validator_" + a.name + attr_name = "__attr_" + a.name + lines.append( + " %s(self, %s, self.%s)" % (val_name, attr_name, a.name) + ) + names_for_globals[val_name] = a.validator + names_for_globals[attr_name] = a + + if post_init: + lines.append("self.__attrs_post_init__()") + + # because this is set only after __attrs_post_init is called, a crash + # will result if post-init tries to access the hash code. This seemed + # preferable to setting this beforehand, in which case alteration to + # field values during post-init combined with post-init accessing the + # hash code would result in silent bugs. + if cache_hash: + if frozen: + if slots: + # if frozen and slots, then _setattr defined above + init_hash_cache = "_setattr('%s', %s)" + else: + # if frozen and not slots, then _inst_dict defined above + init_hash_cache = "_inst_dict['%s'] = %s" + else: + init_hash_cache = "self.%s = %s" + lines.append(init_hash_cache % (_hash_cache_field, "None")) + + # For exceptions we rely on BaseException.__init__ for proper + # initialization. + if is_exc: + vals = ",".join("self." + a.name for a in attrs if a.init) + + lines.append("BaseException.__init__(self, %s)" % (vals,)) + + args = ", ".join(args) + if kw_only_args: + if PY2: + lines = _unpack_kw_only_lines_py2(kw_only_args) + lines + + args += "%s**_kw_only" % (", " if args else "",) # leading comma + else: + args += "%s*, %s" % ( + ", " if args else "", # leading comma + ", ".join(kw_only_args), # kw_only args + ) + return ( + """\ +def __init__(self, {args}): + {lines} +""".format( + args=args, lines="\n ".join(lines) if lines else "pass" + ), + names_for_globals, + annotations, + ) + + +class Attribute(object): + """ + *Read-only* representation of an attribute. + + Instances of this class are frequently used for introspection purposes + like: + + - `fields` returns a tuple of them. + - Validators get them passed as the first argument. + - The *field transformer* hook receives a list of them. + + :attribute name: The name of the attribute. + :attribute inherited: Whether or not that attribute has been inherited from + a base class. + + Plus *all* arguments of `attr.ib` (except for ``factory`` + which is only syntactic sugar for ``default=Factory(...)``. + + .. versionadded:: 20.1.0 *inherited* + .. versionadded:: 20.1.0 *on_setattr* + .. versionchanged:: 20.2.0 *inherited* is not taken into account for + equality checks and hashing anymore. + + For the full version history of the fields, see `attr.ib`. + """ + + __slots__ = ( + "name", + "default", + "validator", + "repr", + "eq", + "order", + "hash", + "init", + "metadata", + "type", + "converter", + "kw_only", + "inherited", + "on_setattr", + ) + + def __init__( + self, + name, + default, + validator, + repr, + cmp, # XXX: unused, remove along with other cmp code. + hash, + init, + inherited, + metadata=None, + type=None, + converter=None, + kw_only=False, + eq=None, + order=None, + on_setattr=None, + ): + eq, order = _determine_eq_order(cmp, eq, order, True) + + # Cache this descriptor here to speed things up later. + bound_setattr = _obj_setattr.__get__(self, Attribute) + + # Despite the big red warning, people *do* instantiate `Attribute` + # themselves. + bound_setattr("name", name) + bound_setattr("default", default) + bound_setattr("validator", validator) + bound_setattr("repr", repr) + bound_setattr("eq", eq) + bound_setattr("order", order) + bound_setattr("hash", hash) + bound_setattr("init", init) + bound_setattr("converter", converter) + bound_setattr( + "metadata", + ( + metadata_proxy(metadata) + if metadata + else _empty_metadata_singleton + ), + ) + bound_setattr("type", type) + bound_setattr("kw_only", kw_only) + bound_setattr("inherited", inherited) + bound_setattr("on_setattr", on_setattr) + + def __setattr__(self, name, value): + raise FrozenInstanceError() + + @classmethod + def from_counting_attr(cls, name, ca, type=None): + # type holds the annotated value. deal with conflicts: + if type is None: + type = ca.type + elif ca.type is not None: + raise ValueError( + "Type annotation and type argument cannot both be present" + ) + inst_dict = { + k: getattr(ca, k) + for k in Attribute.__slots__ + if k + not in ( + "name", + "validator", + "default", + "type", + "inherited", + ) # exclude methods and deprecated alias + } + return cls( + name=name, + validator=ca._validator, + default=ca._default, + type=type, + cmp=None, + inherited=False, + **inst_dict + ) + + @property + def cmp(self): + """ + Simulate the presence of a cmp attribute and warn. + """ + warnings.warn(_CMP_DEPRECATION, DeprecationWarning, stacklevel=2) + + return self.eq and self.order + + # Don't use attr.evolve since fields(Attribute) doesn't work + def evolve(self, **changes): + """ + Copy *self* and apply *changes*. + + This works similarly to `attr.evolve` but that function does not work + with ``Attribute``. + + It is mainly meant to be used for `transform-fields`. + + .. versionadded:: 20.3.0 + """ + new = copy.copy(self) + + new._setattrs(changes.items()) + + return new + + # Don't use _add_pickle since fields(Attribute) doesn't work + def __getstate__(self): + """ + Play nice with pickle. + """ + return tuple( + getattr(self, name) if name != "metadata" else dict(self.metadata) + for name in self.__slots__ + ) + + def __setstate__(self, state): + """ + Play nice with pickle. + """ + self._setattrs(zip(self.__slots__, state)) + + def _setattrs(self, name_values_pairs): + bound_setattr = _obj_setattr.__get__(self, Attribute) + for name, value in name_values_pairs: + if name != "metadata": + bound_setattr(name, value) + else: + bound_setattr( + name, + metadata_proxy(value) + if value + else _empty_metadata_singleton, + ) + + +_a = [ + Attribute( + name=name, + default=NOTHING, + validator=None, + repr=True, + cmp=None, + eq=True, + order=False, + hash=(name != "metadata"), + init=True, + inherited=False, + ) + for name in Attribute.__slots__ +] + +Attribute = _add_hash( + _add_eq( + _add_repr(Attribute, attrs=_a), + attrs=[a for a in _a if a.name != "inherited"], + ), + attrs=[a for a in _a if a.hash and a.name != "inherited"], +) + + +class _CountingAttr(object): + """ + Intermediate representation of attributes that uses a counter to preserve + the order in which the attributes have been defined. + + *Internal* data structure of the attrs library. Running into is most + likely the result of a bug like a forgotten `@attr.s` decorator. + """ + + __slots__ = ( + "counter", + "_default", + "repr", + "eq", + "order", + "hash", + "init", + "metadata", + "_validator", + "converter", + "type", + "kw_only", + "on_setattr", + ) + __attrs_attrs__ = tuple( + Attribute( + name=name, + default=NOTHING, + validator=None, + repr=True, + cmp=None, + hash=True, + init=True, + kw_only=False, + eq=True, + order=False, + inherited=False, + on_setattr=None, + ) + for name in ( + "counter", + "_default", + "repr", + "eq", + "order", + "hash", + "init", + "on_setattr", + ) + ) + ( + Attribute( + name="metadata", + default=None, + validator=None, + repr=True, + cmp=None, + hash=False, + init=True, + kw_only=False, + eq=True, + order=False, + inherited=False, + on_setattr=None, + ), + ) + cls_counter = 0 + + def __init__( + self, + default, + validator, + repr, + cmp, # XXX: unused, remove along with cmp + hash, + init, + converter, + metadata, + type, + kw_only, + eq, + order, + on_setattr, + ): + _CountingAttr.cls_counter += 1 + self.counter = _CountingAttr.cls_counter + self._default = default + self._validator = validator + self.converter = converter + self.repr = repr + self.eq = eq + self.order = order + self.hash = hash + self.init = init + self.metadata = metadata + self.type = type + self.kw_only = kw_only + self.on_setattr = on_setattr + + def validator(self, meth): + """ + Decorator that adds *meth* to the list of validators. + + Returns *meth* unchanged. + + .. versionadded:: 17.1.0 + """ + if self._validator is None: + self._validator = meth + else: + self._validator = and_(self._validator, meth) + return meth + + def default(self, meth): + """ + Decorator that allows to set the default for an attribute. + + Returns *meth* unchanged. + + :raises DefaultAlreadySetError: If default has been set before. + + .. versionadded:: 17.1.0 + """ + if self._default is not NOTHING: + raise DefaultAlreadySetError() + + self._default = Factory(meth, takes_self=True) + + return meth + + +_CountingAttr = _add_eq(_add_repr(_CountingAttr)) + + +@attrs(slots=True, init=False, hash=True) +class Factory(object): + """ + Stores a factory callable. + + If passed as the default value to `attr.ib`, the factory is used to + generate a new value. + + :param callable factory: A callable that takes either none or exactly one + mandatory positional argument depending on *takes_self*. + :param bool takes_self: Pass the partially initialized instance that is + being initialized as a positional argument. + + .. versionadded:: 17.1.0 *takes_self* + """ + + factory = attrib() + takes_self = attrib() + + def __init__(self, factory, takes_self=False): + """ + `Factory` is part of the default machinery so if we want a default + value here, we have to implement it ourselves. + """ + self.factory = factory + self.takes_self = takes_self + + +def make_class(name, attrs, bases=(object,), **attributes_arguments): + """ + A quick way to create a new class called *name* with *attrs*. + + :param str name: The name for the new class. + + :param attrs: A list of names or a dictionary of mappings of names to + attributes. + + If *attrs* is a list or an ordered dict (`dict` on Python 3.6+, + `collections.OrderedDict` otherwise), the order is deduced from + the order of the names or attributes inside *attrs*. Otherwise the + order of the definition of the attributes is used. + :type attrs: `list` or `dict` + + :param tuple bases: Classes that the new class will subclass. + + :param attributes_arguments: Passed unmodified to `attr.s`. + + :return: A new class with *attrs*. + :rtype: type + + .. versionadded:: 17.1.0 *bases* + .. versionchanged:: 18.1.0 If *attrs* is ordered, the order is retained. + """ + if isinstance(attrs, dict): + cls_dict = attrs + elif isinstance(attrs, (list, tuple)): + cls_dict = dict((a, attrib()) for a in attrs) + else: + raise TypeError("attrs argument must be a dict or a list.") + + post_init = cls_dict.pop("__attrs_post_init__", None) + type_ = type( + name, + bases, + {} if post_init is None else {"__attrs_post_init__": post_init}, + ) + # For pickling to work, the __module__ variable needs to be set to the + # frame where the class is created. Bypass this step in environments where + # sys._getframe is not defined (Jython for example) or sys._getframe is not + # defined for arguments greater than 0 (IronPython). + try: + type_.__module__ = sys._getframe(1).f_globals.get( + "__name__", "__main__" + ) + except (AttributeError, ValueError): + pass + + # We do it here for proper warnings with meaningful stacklevel. + cmp = attributes_arguments.pop("cmp", None) + ( + attributes_arguments["eq"], + attributes_arguments["order"], + ) = _determine_eq_order( + cmp, + attributes_arguments.get("eq"), + attributes_arguments.get("order"), + True, + ) + + return _attrs(these=cls_dict, **attributes_arguments)(type_) + + +# These are required by within this module so we define them here and merely +# import into .validators / .converters. + + +@attrs(slots=True, hash=True) +class _AndValidator(object): + """ + Compose many validators to a single one. + """ + + _validators = attrib() + + def __call__(self, inst, attr, value): + for v in self._validators: + v(inst, attr, value) + + +def and_(*validators): + """ + A validator that composes multiple validators into one. + + When called on a value, it runs all wrapped validators. + + :param callables validators: Arbitrary number of validators. + + .. versionadded:: 17.1.0 + """ + vals = [] + for validator in validators: + vals.extend( + validator._validators + if isinstance(validator, _AndValidator) + else [validator] + ) + + return _AndValidator(tuple(vals)) + + +def pipe(*converters): + """ + A converter that composes multiple converters into one. + + When called on a value, it runs all wrapped converters, returning the + *last* value. + + :param callables converters: Arbitrary number of converters. + + .. versionadded:: 20.1.0 + """ + + def pipe_converter(val): + for converter in converters: + val = converter(val) + + return val + + return pipe_converter diff --git a/WebCrawler/venv/Lib/site-packages/attr/_next_gen.py b/WebCrawler/venv/Lib/site-packages/attr/_next_gen.py new file mode 100644 index 0000000..2b5565c --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/attr/_next_gen.py @@ -0,0 +1,160 @@ +""" +This is a Python 3.6 and later-only, keyword-only, and **provisional** API that +calls `attr.s` with different default values. + +Provisional APIs that shall become "import attrs" one glorious day. +""" + +from functools import partial + +from attr.exceptions import UnannotatedAttributeError + +from . import setters +from ._make import NOTHING, _frozen_setattrs, attrib, attrs + + +def define( + maybe_cls=None, + *, + these=None, + repr=None, + hash=None, + init=None, + slots=True, + frozen=False, + weakref_slot=True, + str=False, + auto_attribs=None, + kw_only=False, + cache_hash=False, + auto_exc=True, + eq=None, + order=False, + auto_detect=True, + getstate_setstate=None, + on_setattr=None, + field_transformer=None, +): + r""" + The only behavioral differences are the handling of the *auto_attribs* + option: + + :param Optional[bool] auto_attribs: If set to `True` or `False`, it behaves + exactly like `attr.s`. If left `None`, `attr.s` will try to guess: + + 1. If all attributes are annotated and no `attr.ib` is found, it assumes + *auto_attribs=True*. + 2. Otherwise it assumes *auto_attribs=False* and tries to collect + `attr.ib`\ s. + + and that mutable classes (``frozen=False``) validate on ``__setattr__``. + + .. versionadded:: 20.1.0 + """ + + def do_it(cls, auto_attribs): + return attrs( + maybe_cls=cls, + these=these, + repr=repr, + hash=hash, + init=init, + slots=slots, + frozen=frozen, + weakref_slot=weakref_slot, + str=str, + auto_attribs=auto_attribs, + kw_only=kw_only, + cache_hash=cache_hash, + auto_exc=auto_exc, + eq=eq, + order=order, + auto_detect=auto_detect, + collect_by_mro=True, + getstate_setstate=getstate_setstate, + on_setattr=on_setattr, + field_transformer=field_transformer, + ) + + def wrap(cls): + """ + Making this a wrapper ensures this code runs during class creation. + + We also ensure that frozen-ness of classes is inherited. + """ + nonlocal frozen, on_setattr + + had_on_setattr = on_setattr not in (None, setters.NO_OP) + + # By default, mutable classes validate on setattr. + if frozen is False and on_setattr is None: + on_setattr = setters.validate + + # However, if we subclass a frozen class, we inherit the immutability + # and disable on_setattr. + for base_cls in cls.__bases__: + if base_cls.__setattr__ is _frozen_setattrs: + if had_on_setattr: + raise ValueError( + "Frozen classes can't use on_setattr " + "(frozen-ness was inherited)." + ) + + on_setattr = setters.NO_OP + break + + if auto_attribs is not None: + return do_it(cls, auto_attribs) + + try: + return do_it(cls, True) + except UnannotatedAttributeError: + return do_it(cls, False) + + # maybe_cls's type depends on the usage of the decorator. It's a class + # if it's used as `@attrs` but ``None`` if used as `@attrs()`. + if maybe_cls is None: + return wrap + else: + return wrap(maybe_cls) + + +mutable = define +frozen = partial(define, frozen=True, on_setattr=None) + + +def field( + *, + default=NOTHING, + validator=None, + repr=True, + hash=None, + init=True, + metadata=None, + converter=None, + factory=None, + kw_only=False, + eq=None, + order=None, + on_setattr=None, +): + """ + Identical to `attr.ib`, except keyword-only and with some arguments + removed. + + .. versionadded:: 20.1.0 + """ + return attrib( + default=default, + validator=validator, + repr=repr, + hash=hash, + init=init, + metadata=metadata, + converter=converter, + factory=factory, + kw_only=kw_only, + eq=eq, + order=order, + on_setattr=on_setattr, + ) diff --git a/WebCrawler/venv/Lib/site-packages/attr/_version_info.py b/WebCrawler/venv/Lib/site-packages/attr/_version_info.py new file mode 100644 index 0000000..014e78a --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/attr/_version_info.py @@ -0,0 +1,85 @@ +from __future__ import absolute_import, division, print_function + +from functools import total_ordering + +from ._funcs import astuple +from ._make import attrib, attrs + + +@total_ordering +@attrs(eq=False, order=False, slots=True, frozen=True) +class VersionInfo(object): + """ + A version object that can be compared to tuple of length 1--4: + + >>> attr.VersionInfo(19, 1, 0, "final") <= (19, 2) + True + >>> attr.VersionInfo(19, 1, 0, "final") < (19, 1, 1) + True + >>> vi = attr.VersionInfo(19, 2, 0, "final") + >>> vi < (19, 1, 1) + False + >>> vi < (19,) + False + >>> vi == (19, 2,) + True + >>> vi == (19, 2, 1) + False + + .. versionadded:: 19.2 + """ + + year = attrib(type=int) + minor = attrib(type=int) + micro = attrib(type=int) + releaselevel = attrib(type=str) + + @classmethod + def _from_version_string(cls, s): + """ + Parse *s* and return a _VersionInfo. + """ + v = s.split(".") + if len(v) == 3: + v.append("final") + + return cls( + year=int(v[0]), minor=int(v[1]), micro=int(v[2]), releaselevel=v[3] + ) + + def _ensure_tuple(self, other): + """ + Ensure *other* is a tuple of a valid length. + + Returns a possibly transformed *other* and ourselves as a tuple of + the same length as *other*. + """ + + if self.__class__ is other.__class__: + other = astuple(other) + + if not isinstance(other, tuple): + raise NotImplementedError + + if not (1 <= len(other) <= 4): + raise NotImplementedError + + return astuple(self)[: len(other)], other + + def __eq__(self, other): + try: + us, them = self._ensure_tuple(other) + except NotImplementedError: + return NotImplemented + + return us == them + + def __lt__(self, other): + try: + us, them = self._ensure_tuple(other) + except NotImplementedError: + return NotImplemented + + # Since alphabetically "dev0" < "final" < "post1" < "post2", we don't + # have to do anything special with releaselevel for now. + return us < them diff --git a/WebCrawler/venv/Lib/site-packages/attr/_version_info.pyi b/WebCrawler/venv/Lib/site-packages/attr/_version_info.pyi new file mode 100644 index 0000000..45ced08 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/attr/_version_info.pyi @@ -0,0 +1,9 @@ +class VersionInfo: + @property + def year(self) -> int: ... + @property + def minor(self) -> int: ... + @property + def micro(self) -> int: ... + @property + def releaselevel(self) -> str: ... diff --git a/WebCrawler/venv/Lib/site-packages/attr/converters.py b/WebCrawler/venv/Lib/site-packages/attr/converters.py new file mode 100644 index 0000000..715ce17 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/attr/converters.py @@ -0,0 +1,85 @@ +""" +Commonly useful converters. +""" + +from __future__ import absolute_import, division, print_function + +from ._make import NOTHING, Factory, pipe + + +__all__ = [ + "pipe", + "optional", + "default_if_none", +] + + +def optional(converter): + """ + A converter that allows an attribute to be optional. An optional attribute + is one which can be set to ``None``. + + :param callable converter: the converter that is used for non-``None`` + values. + + .. versionadded:: 17.1.0 + """ + + def optional_converter(val): + if val is None: + return None + return converter(val) + + return optional_converter + + +def default_if_none(default=NOTHING, factory=None): + """ + A converter that allows to replace ``None`` values by *default* or the + result of *factory*. + + :param default: Value to be used if ``None`` is passed. Passing an instance + of `attr.Factory` is supported, however the ``takes_self`` option + is *not*. + :param callable factory: A callable that takes not parameters whose result + is used if ``None`` is passed. + + :raises TypeError: If **neither** *default* or *factory* is passed. + :raises TypeError: If **both** *default* and *factory* are passed. + :raises ValueError: If an instance of `attr.Factory` is passed with + ``takes_self=True``. + + .. versionadded:: 18.2.0 + """ + if default is NOTHING and factory is None: + raise TypeError("Must pass either `default` or `factory`.") + + if default is not NOTHING and factory is not None: + raise TypeError( + "Must pass either `default` or `factory` but not both." + ) + + if factory is not None: + default = Factory(factory) + + if isinstance(default, Factory): + if default.takes_self: + raise ValueError( + "`takes_self` is not supported by default_if_none." + ) + + def default_if_none_converter(val): + if val is not None: + return val + + return default.factory() + + else: + + def default_if_none_converter(val): + if val is not None: + return val + + return default + + return default_if_none_converter diff --git a/WebCrawler/venv/Lib/site-packages/attr/converters.pyi b/WebCrawler/venv/Lib/site-packages/attr/converters.pyi new file mode 100644 index 0000000..7b0caa1 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/attr/converters.pyi @@ -0,0 +1,11 @@ +from typing import TypeVar, Optional, Callable, overload +from . import _ConverterType + +_T = TypeVar("_T") + +def pipe(*validators: _ConverterType) -> _ConverterType: ... +def optional(converter: _ConverterType) -> _ConverterType: ... +@overload +def default_if_none(default: _T) -> _ConverterType: ... +@overload +def default_if_none(*, factory: Callable[[], _T]) -> _ConverterType: ... diff --git a/WebCrawler/venv/Lib/site-packages/attr/exceptions.py b/WebCrawler/venv/Lib/site-packages/attr/exceptions.py new file mode 100644 index 0000000..fcd8910 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/attr/exceptions.py @@ -0,0 +1,92 @@ +from __future__ import absolute_import, division, print_function + + +class FrozenError(AttributeError): + """ + A frozen/immutable instance or attribute haave been attempted to be + modified. + + It mirrors the behavior of ``namedtuples`` by using the same error message + and subclassing `AttributeError`. + + .. versionadded:: 20.1.0 + """ + + msg = "can't set attribute" + args = [msg] + + +class FrozenInstanceError(FrozenError): + """ + A frozen instance has been attempted to be modified. + + .. versionadded:: 16.1.0 + """ + + +class FrozenAttributeError(FrozenError): + """ + A frozen attribute has been attempted to be modified. + + .. versionadded:: 20.1.0 + """ + + +class AttrsAttributeNotFoundError(ValueError): + """ + An ``attrs`` function couldn't find an attribute that the user asked for. + + .. versionadded:: 16.2.0 + """ + + +class NotAnAttrsClassError(ValueError): + """ + A non-``attrs`` class has been passed into an ``attrs`` function. + + .. versionadded:: 16.2.0 + """ + + +class DefaultAlreadySetError(RuntimeError): + """ + A default has been set using ``attr.ib()`` and is attempted to be reset + using the decorator. + + .. versionadded:: 17.1.0 + """ + + +class UnannotatedAttributeError(RuntimeError): + """ + A class with ``auto_attribs=True`` has an ``attr.ib()`` without a type + annotation. + + .. versionadded:: 17.3.0 + """ + + +class PythonTooOldError(RuntimeError): + """ + It was attempted to use an ``attrs`` feature that requires a newer Python + version. + + .. versionadded:: 18.2.0 + """ + + +class NotCallableError(TypeError): + """ + A ``attr.ib()`` requiring a callable has been set with a value + that is not callable. + + .. versionadded:: 19.2.0 + """ + + def __init__(self, msg, value): + super(TypeError, self).__init__(msg, value) + self.msg = msg + self.value = value + + def __str__(self): + return str(self.msg) diff --git a/WebCrawler/venv/Lib/site-packages/attr/exceptions.pyi b/WebCrawler/venv/Lib/site-packages/attr/exceptions.pyi new file mode 100644 index 0000000..f268011 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/attr/exceptions.pyi @@ -0,0 +1,17 @@ +from typing import Any + +class FrozenError(AttributeError): + msg: str = ... + +class FrozenInstanceError(FrozenError): ... +class FrozenAttributeError(FrozenError): ... +class AttrsAttributeNotFoundError(ValueError): ... +class NotAnAttrsClassError(ValueError): ... +class DefaultAlreadySetError(RuntimeError): ... +class UnannotatedAttributeError(RuntimeError): ... +class PythonTooOldError(RuntimeError): ... + +class NotCallableError(TypeError): + msg: str = ... + value: Any = ... + def __init__(self, msg: str, value: Any) -> None: ... diff --git a/WebCrawler/venv/Lib/site-packages/attr/filters.py b/WebCrawler/venv/Lib/site-packages/attr/filters.py new file mode 100644 index 0000000..dc47e8f --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/attr/filters.py @@ -0,0 +1,52 @@ +""" +Commonly useful filters for `attr.asdict`. +""" + +from __future__ import absolute_import, division, print_function + +from ._compat import isclass +from ._make import Attribute + + +def _split_what(what): + """ + Returns a tuple of `frozenset`s of classes and attributes. + """ + return ( + frozenset(cls for cls in what if isclass(cls)), + frozenset(cls for cls in what if isinstance(cls, Attribute)), + ) + + +def include(*what): + """ + Whitelist *what*. + + :param what: What to whitelist. + :type what: `list` of `type` or `attr.Attribute`\\ s + + :rtype: `callable` + """ + cls, attrs = _split_what(what) + + def include_(attribute, value): + return value.__class__ in cls or attribute in attrs + + return include_ + + +def exclude(*what): + """ + Blacklist *what*. + + :param what: What to blacklist. + :type what: `list` of classes or `attr.Attribute`\\ s. + + :rtype: `callable` + """ + cls, attrs = _split_what(what) + + def exclude_(attribute, value): + return value.__class__ not in cls and attribute not in attrs + + return exclude_ diff --git a/WebCrawler/venv/Lib/site-packages/attr/filters.pyi b/WebCrawler/venv/Lib/site-packages/attr/filters.pyi new file mode 100644 index 0000000..68368fe --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/attr/filters.pyi @@ -0,0 +1,5 @@ +from typing import Union, Any +from . import Attribute, _FilterType + +def include(*what: Union[type, Attribute[Any]]) -> _FilterType[Any]: ... +def exclude(*what: Union[type, Attribute[Any]]) -> _FilterType[Any]: ... diff --git a/WebCrawler/venv/Lib/site-packages/attr/py.typed b/WebCrawler/venv/Lib/site-packages/attr/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/WebCrawler/venv/Lib/site-packages/attr/setters.py b/WebCrawler/venv/Lib/site-packages/attr/setters.py new file mode 100644 index 0000000..240014b --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/attr/setters.py @@ -0,0 +1,77 @@ +""" +Commonly used hooks for on_setattr. +""" + +from __future__ import absolute_import, division, print_function + +from . import _config +from .exceptions import FrozenAttributeError + + +def pipe(*setters): + """ + Run all *setters* and return the return value of the last one. + + .. versionadded:: 20.1.0 + """ + + def wrapped_pipe(instance, attrib, new_value): + rv = new_value + + for setter in setters: + rv = setter(instance, attrib, rv) + + return rv + + return wrapped_pipe + + +def frozen(_, __, ___): + """ + Prevent an attribute to be modified. + + .. versionadded:: 20.1.0 + """ + raise FrozenAttributeError() + + +def validate(instance, attrib, new_value): + """ + Run *attrib*'s validator on *new_value* if it has one. + + .. versionadded:: 20.1.0 + """ + if _config._run_validators is False: + return new_value + + v = attrib.validator + if not v: + return new_value + + v(instance, attrib, new_value) + + return new_value + + +def convert(instance, attrib, new_value): + """ + Run *attrib*'s converter -- if it has one -- on *new_value* and return the + result. + + .. versionadded:: 20.1.0 + """ + c = attrib.converter + if c: + return c(new_value) + + return new_value + + +NO_OP = object() +""" +Sentinel for disabling class-wide *on_setattr* hooks for certain attributes. + +Does not work in `pipe` or within lists. + +.. versionadded:: 20.1.0 +""" diff --git a/WebCrawler/venv/Lib/site-packages/attr/setters.pyi b/WebCrawler/venv/Lib/site-packages/attr/setters.pyi new file mode 100644 index 0000000..19bc33f --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/attr/setters.pyi @@ -0,0 +1,18 @@ +from . import _OnSetAttrType, Attribute +from typing import TypeVar, Any, NewType, NoReturn, cast + +_T = TypeVar("_T") + +def frozen( + instance: Any, attribute: Attribute, new_value: Any +) -> NoReturn: ... +def pipe(*setters: _OnSetAttrType) -> _OnSetAttrType: ... +def validate(instance: Any, attribute: Attribute[_T], new_value: _T) -> _T: ... + +# convert is allowed to return Any, because they can be chained using pipe. +def convert( + instance: Any, attribute: Attribute[Any], new_value: Any +) -> Any: ... + +_NoOpType = NewType("_NoOpType", object) +NO_OP: _NoOpType diff --git a/WebCrawler/venv/Lib/site-packages/attr/validators.py b/WebCrawler/venv/Lib/site-packages/attr/validators.py new file mode 100644 index 0000000..b9a7305 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/attr/validators.py @@ -0,0 +1,379 @@ +""" +Commonly useful validators. +""" + +from __future__ import absolute_import, division, print_function + +import re + +from ._make import _AndValidator, and_, attrib, attrs +from .exceptions import NotCallableError + + +__all__ = [ + "and_", + "deep_iterable", + "deep_mapping", + "in_", + "instance_of", + "is_callable", + "matches_re", + "optional", + "provides", +] + + +@attrs(repr=False, slots=True, hash=True) +class _InstanceOfValidator(object): + type = attrib() + + def __call__(self, inst, attr, value): + """ + We use a callable class to be able to change the ``__repr__``. + """ + if not isinstance(value, self.type): + raise TypeError( + "'{name}' must be {type!r} (got {value!r} that is a " + "{actual!r}).".format( + name=attr.name, + type=self.type, + actual=value.__class__, + value=value, + ), + attr, + self.type, + value, + ) + + def __repr__(self): + return "".format( + type=self.type + ) + + +def instance_of(type): + """ + A validator that raises a `TypeError` if the initializer is called + with a wrong type for this particular attribute (checks are performed using + `isinstance` therefore it's also valid to pass a tuple of types). + + :param type: The type to check for. + :type type: type or tuple of types + + :raises TypeError: With a human readable error message, the attribute + (of type `attr.Attribute`), the expected type, and the value it + got. + """ + return _InstanceOfValidator(type) + + +@attrs(repr=False, frozen=True, slots=True) +class _MatchesReValidator(object): + regex = attrib() + flags = attrib() + match_func = attrib() + + def __call__(self, inst, attr, value): + """ + We use a callable class to be able to change the ``__repr__``. + """ + if not self.match_func(value): + raise ValueError( + "'{name}' must match regex {regex!r}" + " ({value!r} doesn't)".format( + name=attr.name, regex=self.regex.pattern, value=value + ), + attr, + self.regex, + value, + ) + + def __repr__(self): + return "".format( + regex=self.regex + ) + + +def matches_re(regex, flags=0, func=None): + r""" + A validator that raises `ValueError` if the initializer is called + with a string that doesn't match *regex*. + + :param str regex: a regex string to match against + :param int flags: flags that will be passed to the underlying re function + (default 0) + :param callable func: which underlying `re` function to call (options + are `re.fullmatch`, `re.search`, `re.match`, default + is ``None`` which means either `re.fullmatch` or an emulation of + it on Python 2). For performance reasons, they won't be used directly + but on a pre-`re.compile`\ ed pattern. + + .. versionadded:: 19.2.0 + """ + fullmatch = getattr(re, "fullmatch", None) + valid_funcs = (fullmatch, None, re.search, re.match) + if func not in valid_funcs: + raise ValueError( + "'func' must be one of %s." + % ( + ", ".join( + sorted( + e and e.__name__ or "None" for e in set(valid_funcs) + ) + ), + ) + ) + + pattern = re.compile(regex, flags) + if func is re.match: + match_func = pattern.match + elif func is re.search: + match_func = pattern.search + else: + if fullmatch: + match_func = pattern.fullmatch + else: + pattern = re.compile(r"(?:{})\Z".format(regex), flags) + match_func = pattern.match + + return _MatchesReValidator(pattern, flags, match_func) + + +@attrs(repr=False, slots=True, hash=True) +class _ProvidesValidator(object): + interface = attrib() + + def __call__(self, inst, attr, value): + """ + We use a callable class to be able to change the ``__repr__``. + """ + if not self.interface.providedBy(value): + raise TypeError( + "'{name}' must provide {interface!r} which {value!r} " + "doesn't.".format( + name=attr.name, interface=self.interface, value=value + ), + attr, + self.interface, + value, + ) + + def __repr__(self): + return "".format( + interface=self.interface + ) + + +def provides(interface): + """ + A validator that raises a `TypeError` if the initializer is called + with an object that does not provide the requested *interface* (checks are + performed using ``interface.providedBy(value)`` (see `zope.interface + `_). + + :param interface: The interface to check for. + :type interface: ``zope.interface.Interface`` + + :raises TypeError: With a human readable error message, the attribute + (of type `attr.Attribute`), the expected interface, and the + value it got. + """ + return _ProvidesValidator(interface) + + +@attrs(repr=False, slots=True, hash=True) +class _OptionalValidator(object): + validator = attrib() + + def __call__(self, inst, attr, value): + if value is None: + return + + self.validator(inst, attr, value) + + def __repr__(self): + return "".format( + what=repr(self.validator) + ) + + +def optional(validator): + """ + A validator that makes an attribute optional. An optional attribute is one + which can be set to ``None`` in addition to satisfying the requirements of + the sub-validator. + + :param validator: A validator (or a list of validators) that is used for + non-``None`` values. + :type validator: callable or `list` of callables. + + .. versionadded:: 15.1.0 + .. versionchanged:: 17.1.0 *validator* can be a list of validators. + """ + if isinstance(validator, list): + return _OptionalValidator(_AndValidator(validator)) + return _OptionalValidator(validator) + + +@attrs(repr=False, slots=True, hash=True) +class _InValidator(object): + options = attrib() + + def __call__(self, inst, attr, value): + try: + in_options = value in self.options + except TypeError: # e.g. `1 in "abc"` + in_options = False + + if not in_options: + raise ValueError( + "'{name}' must be in {options!r} (got {value!r})".format( + name=attr.name, options=self.options, value=value + ) + ) + + def __repr__(self): + return "".format( + options=self.options + ) + + +def in_(options): + """ + A validator that raises a `ValueError` if the initializer is called + with a value that does not belong in the options provided. The check is + performed using ``value in options``. + + :param options: Allowed options. + :type options: list, tuple, `enum.Enum`, ... + + :raises ValueError: With a human readable error message, the attribute (of + type `attr.Attribute`), the expected options, and the value it + got. + + .. versionadded:: 17.1.0 + """ + return _InValidator(options) + + +@attrs(repr=False, slots=False, hash=True) +class _IsCallableValidator(object): + def __call__(self, inst, attr, value): + """ + We use a callable class to be able to change the ``__repr__``. + """ + if not callable(value): + message = ( + "'{name}' must be callable " + "(got {value!r} that is a {actual!r})." + ) + raise NotCallableError( + msg=message.format( + name=attr.name, value=value, actual=value.__class__ + ), + value=value, + ) + + def __repr__(self): + return "" + + +def is_callable(): + """ + A validator that raises a `attr.exceptions.NotCallableError` if the + initializer is called with a value for this particular attribute + that is not callable. + + .. versionadded:: 19.1.0 + + :raises `attr.exceptions.NotCallableError`: With a human readable error + message containing the attribute (`attr.Attribute`) name, + and the value it got. + """ + return _IsCallableValidator() + + +@attrs(repr=False, slots=True, hash=True) +class _DeepIterable(object): + member_validator = attrib(validator=is_callable()) + iterable_validator = attrib( + default=None, validator=optional(is_callable()) + ) + + def __call__(self, inst, attr, value): + """ + We use a callable class to be able to change the ``__repr__``. + """ + if self.iterable_validator is not None: + self.iterable_validator(inst, attr, value) + + for member in value: + self.member_validator(inst, attr, member) + + def __repr__(self): + iterable_identifier = ( + "" + if self.iterable_validator is None + else " {iterable!r}".format(iterable=self.iterable_validator) + ) + return ( + "" + ).format( + iterable_identifier=iterable_identifier, + member=self.member_validator, + ) + + +def deep_iterable(member_validator, iterable_validator=None): + """ + A validator that performs deep validation of an iterable. + + :param member_validator: Validator to apply to iterable members + :param iterable_validator: Validator to apply to iterable itself + (optional) + + .. versionadded:: 19.1.0 + + :raises TypeError: if any sub-validators fail + """ + return _DeepIterable(member_validator, iterable_validator) + + +@attrs(repr=False, slots=True, hash=True) +class _DeepMapping(object): + key_validator = attrib(validator=is_callable()) + value_validator = attrib(validator=is_callable()) + mapping_validator = attrib(default=None, validator=optional(is_callable())) + + def __call__(self, inst, attr, value): + """ + We use a callable class to be able to change the ``__repr__``. + """ + if self.mapping_validator is not None: + self.mapping_validator(inst, attr, value) + + for key in value: + self.key_validator(inst, attr, key) + self.value_validator(inst, attr, value[key]) + + def __repr__(self): + return ( + "" + ).format(key=self.key_validator, value=self.value_validator) + + +def deep_mapping(key_validator, value_validator, mapping_validator=None): + """ + A validator that performs deep validation of a dictionary. + + :param key_validator: Validator to apply to dictionary keys + :param value_validator: Validator to apply to dictionary values + :param mapping_validator: Validator to apply to top-level mapping + attribute (optional) + + .. versionadded:: 19.1.0 + + :raises TypeError: if any sub-validators fail + """ + return _DeepMapping(key_validator, value_validator, mapping_validator) diff --git a/WebCrawler/venv/Lib/site-packages/attr/validators.pyi b/WebCrawler/venv/Lib/site-packages/attr/validators.pyi new file mode 100644 index 0000000..9a22abb --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/attr/validators.pyi @@ -0,0 +1,66 @@ +from typing import ( + Container, + List, + Union, + TypeVar, + Type, + Any, + Optional, + Tuple, + Iterable, + Mapping, + Callable, + Match, + AnyStr, + overload, +) +from . import _ValidatorType + +_T = TypeVar("_T") +_T1 = TypeVar("_T1") +_T2 = TypeVar("_T2") +_T3 = TypeVar("_T3") +_I = TypeVar("_I", bound=Iterable) +_K = TypeVar("_K") +_V = TypeVar("_V") +_M = TypeVar("_M", bound=Mapping) + +# To be more precise on instance_of use some overloads. +# If there are more than 3 items in the tuple then we fall back to Any +@overload +def instance_of(type: Type[_T]) -> _ValidatorType[_T]: ... +@overload +def instance_of(type: Tuple[Type[_T]]) -> _ValidatorType[_T]: ... +@overload +def instance_of( + type: Tuple[Type[_T1], Type[_T2]] +) -> _ValidatorType[Union[_T1, _T2]]: ... +@overload +def instance_of( + type: Tuple[Type[_T1], Type[_T2], Type[_T3]] +) -> _ValidatorType[Union[_T1, _T2, _T3]]: ... +@overload +def instance_of(type: Tuple[type, ...]) -> _ValidatorType[Any]: ... +def provides(interface: Any) -> _ValidatorType[Any]: ... +def optional( + validator: Union[_ValidatorType[_T], List[_ValidatorType[_T]]] +) -> _ValidatorType[Optional[_T]]: ... +def in_(options: Container[_T]) -> _ValidatorType[_T]: ... +def and_(*validators: _ValidatorType[_T]) -> _ValidatorType[_T]: ... +def matches_re( + regex: AnyStr, + flags: int = ..., + func: Optional[ + Callable[[AnyStr, AnyStr, int], Optional[Match[AnyStr]]] + ] = ..., +) -> _ValidatorType[AnyStr]: ... +def deep_iterable( + member_validator: _ValidatorType[_T], + iterable_validator: Optional[_ValidatorType[_I]] = ..., +) -> _ValidatorType[_I]: ... +def deep_mapping( + key_validator: _ValidatorType[_K], + value_validator: _ValidatorType[_V], + mapping_validator: Optional[_ValidatorType[_M]] = ..., +) -> _ValidatorType[_M]: ... +def is_callable() -> _ValidatorType[_T]: ... diff --git a/WebCrawler/venv/Lib/site-packages/attrs-20.3.0.dist-info/AUTHORS.rst b/WebCrawler/venv/Lib/site-packages/attrs-20.3.0.dist-info/AUTHORS.rst new file mode 100644 index 0000000..f14ef6c --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/attrs-20.3.0.dist-info/AUTHORS.rst @@ -0,0 +1,11 @@ +Credits +======= + +``attrs`` is written and maintained by `Hynek Schlawack `_. + +The development is kindly supported by `Variomedia AG `_. + +A full list of contributors can be found in `GitHub's overview `_. + +It’s the spiritual successor of `characteristic `_ and aspires to fix some of it clunkiness and unfortunate decisions. +Both were inspired by Twisted’s `FancyEqMixin `_ but both are implemented using class decorators because `subclassing is bad for you `_, m’kay? diff --git a/WebCrawler/venv/Lib/site-packages/attrs-20.3.0.dist-info/INSTALLER b/WebCrawler/venv/Lib/site-packages/attrs-20.3.0.dist-info/INSTALLER new file mode 100644 index 0000000..a1b589e --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/attrs-20.3.0.dist-info/INSTALLER @@ -0,0 +1 @@ +pip diff --git a/WebCrawler/venv/Lib/site-packages/attrs-20.3.0.dist-info/LICENSE b/WebCrawler/venv/Lib/site-packages/attrs-20.3.0.dist-info/LICENSE new file mode 100644 index 0000000..7ae3df9 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/attrs-20.3.0.dist-info/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2015 Hynek Schlawack + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/WebCrawler/venv/Lib/site-packages/attrs-20.3.0.dist-info/METADATA b/WebCrawler/venv/Lib/site-packages/attrs-20.3.0.dist-info/METADATA new file mode 100644 index 0000000..a92cacb --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/attrs-20.3.0.dist-info/METADATA @@ -0,0 +1,241 @@ +Metadata-Version: 2.1 +Name: attrs +Version: 20.3.0 +Summary: Classes Without Boilerplate +Home-page: https://www.attrs.org/ +Author: Hynek Schlawack +Author-email: hs@ox.cx +Maintainer: Hynek Schlawack +Maintainer-email: hs@ox.cx +License: MIT +Project-URL: Documentation, https://www.attrs.org/ +Project-URL: Bug Tracker, https://github.com/python-attrs/attrs/issues +Project-URL: Source Code, https://github.com/python-attrs/attrs +Project-URL: Funding, https://github.com/sponsors/hynek +Project-URL: Tidelift, https://tidelift.com/subscription/pkg/pypi-attrs?utm_source=pypi-attrs&utm_medium=pypi +Keywords: class,attribute,boilerplate +Platform: UNKNOWN +Classifier: Development Status :: 5 - Production/Stable +Classifier: Intended Audience :: Developers +Classifier: Natural Language :: English +Classifier: License :: OSI Approved :: MIT License +Classifier: Operating System :: OS Independent +Classifier: Programming Language :: Python +Classifier: Programming Language :: Python :: 2 +Classifier: Programming Language :: Python :: 2.7 +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3.5 +Classifier: Programming Language :: Python :: 3.6 +Classifier: Programming Language :: Python :: 3.7 +Classifier: Programming Language :: Python :: 3.8 +Classifier: Programming Language :: Python :: 3.9 +Classifier: Programming Language :: Python :: Implementation :: CPython +Classifier: Programming Language :: Python :: Implementation :: PyPy +Classifier: Topic :: Software Development :: Libraries :: Python Modules +Requires-Python: >=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.* +Description-Content-Type: text/x-rst +Provides-Extra: dev +Requires-Dist: coverage[toml] (>=5.0.2) ; extra == 'dev' +Requires-Dist: hypothesis ; extra == 'dev' +Requires-Dist: pympler ; extra == 'dev' +Requires-Dist: pytest (>=4.3.0) ; extra == 'dev' +Requires-Dist: six ; extra == 'dev' +Requires-Dist: zope.interface ; extra == 'dev' +Requires-Dist: furo ; extra == 'dev' +Requires-Dist: sphinx ; extra == 'dev' +Requires-Dist: pre-commit ; extra == 'dev' +Provides-Extra: docs +Requires-Dist: furo ; extra == 'docs' +Requires-Dist: sphinx ; extra == 'docs' +Requires-Dist: zope.interface ; extra == 'docs' +Provides-Extra: tests +Requires-Dist: coverage[toml] (>=5.0.2) ; extra == 'tests' +Requires-Dist: hypothesis ; extra == 'tests' +Requires-Dist: pympler ; extra == 'tests' +Requires-Dist: pytest (>=4.3.0) ; extra == 'tests' +Requires-Dist: six ; extra == 'tests' +Requires-Dist: zope.interface ; extra == 'tests' +Provides-Extra: tests_no_zope +Requires-Dist: coverage[toml] (>=5.0.2) ; extra == 'tests_no_zope' +Requires-Dist: hypothesis ; extra == 'tests_no_zope' +Requires-Dist: pympler ; extra == 'tests_no_zope' +Requires-Dist: pytest (>=4.3.0) ; extra == 'tests_no_zope' +Requires-Dist: six ; extra == 'tests_no_zope' + +.. image:: https://www.attrs.org/en/latest/_static/attrs_logo.png + :alt: attrs Logo + +====================================== +``attrs``: Classes Without Boilerplate +====================================== + +.. image:: https://readthedocs.org/projects/attrs/badge/?version=stable + :target: https://www.attrs.org/en/stable/?badge=stable + :alt: Documentation Status + +.. image:: https://github.com/python-attrs/attrs/workflows/CI/badge.svg?branch=master + :target: https://github.com/python-attrs/attrs/actions?workflow=CI + :alt: CI Status + +.. image:: https://codecov.io/github/python-attrs/attrs/branch/master/graph/badge.svg + :target: https://codecov.io/github/python-attrs/attrs + :alt: Test Coverage + +.. image:: https://img.shields.io/badge/code%20style-black-000000.svg + :target: https://github.com/psf/black + :alt: Code style: black + +.. teaser-begin + +``attrs`` is the Python package that will bring back the **joy** of **writing classes** by relieving you from the drudgery of implementing object protocols (aka `dunder `_ methods). + +Its main goal is to help you to write **concise** and **correct** software without slowing down your code. + +.. teaser-end + +For that, it gives you a class decorator and a way to declaratively define the attributes on that class: + +.. -code-begin- + +.. code-block:: pycon + + >>> import attr + + >>> @attr.s + ... class SomeClass(object): + ... a_number = attr.ib(default=42) + ... list_of_numbers = attr.ib(factory=list) + ... + ... def hard_math(self, another_number): + ... return self.a_number + sum(self.list_of_numbers) * another_number + + + >>> sc = SomeClass(1, [1, 2, 3]) + >>> sc + SomeClass(a_number=1, list_of_numbers=[1, 2, 3]) + + >>> sc.hard_math(3) + 19 + >>> sc == SomeClass(1, [1, 2, 3]) + True + >>> sc != SomeClass(2, [3, 2, 1]) + True + + >>> attr.asdict(sc) + {'a_number': 1, 'list_of_numbers': [1, 2, 3]} + + >>> SomeClass() + SomeClass(a_number=42, list_of_numbers=[]) + + >>> C = attr.make_class("C", ["a", "b"]) + >>> C("foo", "bar") + C(a='foo', b='bar') + + +After *declaring* your attributes ``attrs`` gives you: + +- a concise and explicit overview of the class's attributes, +- a nice human-readable ``__repr__``, +- a complete set of comparison methods (equality and ordering), +- an initializer, +- and much more, + +*without* writing dull boilerplate code again and again and *without* runtime performance penalties. + +On Python 3.6 and later, you can often even drop the calls to ``attr.ib()`` by using `type annotations `_. + +This gives you the power to use actual classes with actual types in your code instead of confusing ``tuple``\ s or `confusingly behaving `_ ``namedtuple``\ s. +Which in turn encourages you to write *small classes* that do `one thing well `_. +Never again violate the `single responsibility principle `_ just because implementing ``__init__`` et al is a painful drag. + + +.. -getting-help- + +Getting Help +============ + +Please use the ``python-attrs`` tag on `StackOverflow `_ to get help. + +Answering questions of your fellow developers is also a great way to help the project! + + +.. -project-information- + +Project Information +=================== + +``attrs`` is released under the `MIT `_ license, +its documentation lives at `Read the Docs `_, +the code on `GitHub `_, +and the latest release on `PyPI `_. +It’s rigorously tested on Python 2.7, 3.5+, and PyPy. + +We collect information on **third-party extensions** in our `wiki `_. +Feel free to browse and add your own! + +If you'd like to contribute to ``attrs`` you're most welcome and we've written `a little guide `_ to get you started! + + +``attrs`` for Enterprise +------------------------ + +Available as part of the Tidelift Subscription. + +The maintainers of ``attrs`` and thousands of other packages are working with Tidelift to deliver commercial support and maintenance for the open source packages you use to build your applications. +Save time, reduce risk, and improve code health, while paying the maintainers of the exact packages you use. +`Learn more. `_ + + +Release Information +=================== + +20.3.0 (2020-11-05) +------------------- + +Backward-incompatible Changes +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +- ``attr.define()``, ``attr.frozen()``, ``attr.mutable()``, and ``attr.field()`` remain **provisional**. + + This release does **not** change change anything about them and they are already used widely in production though. + + If you wish to use them together with mypy, you can simply drop `this plugin `_ into your project. + + Feel free to provide feedback to them in the linked issue #668. + + We will release the ``attrs`` namespace once we have the feeling that the APIs have properly settled. + `#668 `_ + + +Changes +^^^^^^^ + +- ``attr.s()`` now has a *field_transformer* hook that is called for all ``Attribute``\ s and returns a (modified or updated) list of ``Attribute`` instances. + ``attr.asdict()`` has a *value_serializer* hook that can change the way values are converted. + Both hooks are meant to help with data (de-)serialization workflows. + `#653 `_ +- ``kw_only=True`` now works on Python 2. + `#700 `_ +- ``raise from`` now works on frozen classes on PyPy. + `#703 `_, + `#712 `_ +- ``attr.asdict()`` and ``attr.astuple()`` now treat ``frozenset``\ s like ``set``\ s with regards to the *retain_collection_types* argument. + `#704 `_ +- The type stubs for ``attr.s()`` and ``attr.make_class()`` are not missing the *collect_by_mro* argument anymore. + `#711 `_ + +`Full changelog `_. + +Credits +======= + +``attrs`` is written and maintained by `Hynek Schlawack `_. + +The development is kindly supported by `Variomedia AG `_. + +A full list of contributors can be found in `GitHub's overview `_. + +It’s the spiritual successor of `characteristic `_ and aspires to fix some of it clunkiness and unfortunate decisions. +Both were inspired by Twisted’s `FancyEqMixin `_ but both are implemented using class decorators because `subclassing is bad for you `_, m’kay? + + diff --git a/WebCrawler/venv/Lib/site-packages/attrs-20.3.0.dist-info/RECORD b/WebCrawler/venv/Lib/site-packages/attrs-20.3.0.dist-info/RECORD new file mode 100644 index 0000000..1e64401 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/attrs-20.3.0.dist-info/RECORD @@ -0,0 +1,39 @@ +attr/__init__.py,sha256=70KmZOgz2sUvtRTC_IuXEeN2ttOyBWHn4XA59aqGXPs,1568 +attr/__init__.pyi,sha256=ca_4sg7z0e_EL7ehy-flXVGAju5PBX2hVo51dUmPMi0,12986 +attr/__pycache__/__init__.cpython-39.pyc,, +attr/__pycache__/_compat.cpython-39.pyc,, +attr/__pycache__/_config.cpython-39.pyc,, +attr/__pycache__/_funcs.cpython-39.pyc,, +attr/__pycache__/_make.cpython-39.pyc,, +attr/__pycache__/_next_gen.cpython-39.pyc,, +attr/__pycache__/_version_info.cpython-39.pyc,, +attr/__pycache__/converters.cpython-39.pyc,, +attr/__pycache__/exceptions.cpython-39.pyc,, +attr/__pycache__/filters.cpython-39.pyc,, +attr/__pycache__/setters.cpython-39.pyc,, +attr/__pycache__/validators.cpython-39.pyc,, +attr/_compat.py,sha256=rZhpP09xbyWSzMv796XQbryIr21oReJFvA70G3lrHxg,7308 +attr/_config.py,sha256=_KvW0mQdH2PYjHc0YfIUaV_o2pVfM7ziMEYTxwmEhOA,514 +attr/_funcs.py,sha256=PvFQlflEswO_qIR2sUr4a4x8ggQpEoDKe3YKM2rLJu4,13081 +attr/_make.py,sha256=61XB4-SHQpFbWbStGWotTTbzVT2m49DUovRgnxpMqmU,88313 +attr/_next_gen.py,sha256=x6TU2rVOXmFmrNNvkfshJsxyRbAAK0wDI4SJV2OI97c,4138 +attr/_version_info.py,sha256=azMi1lNelb3cJvvYUMXsXVbUANkRzbD5IEiaXVpeVr4,2162 +attr/_version_info.pyi,sha256=x_M3L3WuB7r_ULXAWjx959udKQ4HLB8l-hsc1FDGNvk,209 +attr/converters.py,sha256=CaK6iLtEMmemrqU8LQ1D2nWtbo9dGPAv4UaZ0rFzhOA,2214 +attr/converters.pyi,sha256=fVGSfawF3NMy2EBApkC7dAwMuujWCHnGEnnAgsbkVpg,380 +attr/exceptions.py,sha256=gmlET97ikqdQVvy7Ff9p7zVvqc2SsNtTd-r30pva1GE,1950 +attr/exceptions.pyi,sha256=zZq8bCUnKAy9mDtBEw42ZhPhAUIHoTKedDQInJD883M,539 +attr/filters.py,sha256=weDxwATsa69T_0bPVjiM1fGsciAMQmwhY5G8Jm5BxuI,1098 +attr/filters.pyi,sha256=xDpmKQlFdssgxGa5tsl1ADh_3zwAwAT4vUhd8h-8-Tk,214 +attr/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +attr/setters.py,sha256=0ElzHwdVK3dsYcQi2CXkFvhx8fNxUI5OVhw8SWeaKmA,1434 +attr/setters.pyi,sha256=SYr6adhx4f0dSkmmBICg6eK8WMev5jT-KJQJTdul078,567 +attr/validators.py,sha256=6DBx1jt4oZxx1ppvx6JWqm9-UAsYpXC4HTwxJilCeRg,11497 +attr/validators.pyi,sha256=vZgsJqUwrJevh4v_Hd7_RSXqDrBctE6-3AEZ7uYKodo,1868 +attrs-20.3.0.dist-info/AUTHORS.rst,sha256=wsqCNbGz_mklcJrt54APIZHZpoTIJLkXqEhhn4Nd8hc,752 +attrs-20.3.0.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4 +attrs-20.3.0.dist-info/LICENSE,sha256=v2WaKLSSQGAvVrvfSQy-LsUJsVuY-Z17GaUsdA4yeGM,1082 +attrs-20.3.0.dist-info/METADATA,sha256=2XTmALrRRbIZj9J8pJgpKYnyATu_NAL8vfUnqRFpE5w,10220 +attrs-20.3.0.dist-info/RECORD,, +attrs-20.3.0.dist-info/WHEEL,sha256=ADKeyaGyKF5DwBNE0sRE5pvW-bSkFMJfBuhzZ3rceP4,110 +attrs-20.3.0.dist-info/top_level.txt,sha256=tlRYMddkRlKPqJ96wP2_j9uEsmcNHgD2SbuWd4CzGVU,5 diff --git a/WebCrawler/venv/Lib/site-packages/attrs-20.3.0.dist-info/WHEEL b/WebCrawler/venv/Lib/site-packages/attrs-20.3.0.dist-info/WHEEL new file mode 100644 index 0000000..6d38aa0 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/attrs-20.3.0.dist-info/WHEEL @@ -0,0 +1,6 @@ +Wheel-Version: 1.0 +Generator: bdist_wheel (0.35.1) +Root-Is-Purelib: true +Tag: py2-none-any +Tag: py3-none-any + diff --git a/WebCrawler/venv/Lib/site-packages/attrs-20.3.0.dist-info/top_level.txt b/WebCrawler/venv/Lib/site-packages/attrs-20.3.0.dist-info/top_level.txt new file mode 100644 index 0000000..66a062d --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/attrs-20.3.0.dist-info/top_level.txt @@ -0,0 +1 @@ +attr diff --git a/WebCrawler/venv/Lib/site-packages/automat/__init__.py b/WebCrawler/venv/Lib/site-packages/automat/__init__.py new file mode 100644 index 0000000..570b84f --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/automat/__init__.py @@ -0,0 +1,8 @@ +# -*- test-case-name: automat -*- +from ._methodical import MethodicalMachine +from ._core import NoTransition + +__all__ = [ + 'MethodicalMachine', + 'NoTransition', +] diff --git a/WebCrawler/venv/Lib/site-packages/automat/__pycache__/__init__.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/automat/__pycache__/__init__.cpython-39.pyc new file mode 100644 index 0000000..07f8dfb Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/automat/__pycache__/__init__.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/automat/__pycache__/_core.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/automat/__pycache__/_core.cpython-39.pyc new file mode 100644 index 0000000..6233e5d Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/automat/__pycache__/_core.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/automat/__pycache__/_discover.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/automat/__pycache__/_discover.cpython-39.pyc new file mode 100644 index 0000000..3e3a894 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/automat/__pycache__/_discover.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/automat/__pycache__/_introspection.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/automat/__pycache__/_introspection.cpython-39.pyc new file mode 100644 index 0000000..83050f1 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/automat/__pycache__/_introspection.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/automat/__pycache__/_methodical.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/automat/__pycache__/_methodical.cpython-39.pyc new file mode 100644 index 0000000..58419c0 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/automat/__pycache__/_methodical.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/automat/__pycache__/_visualize.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/automat/__pycache__/_visualize.cpython-39.pyc new file mode 100644 index 0000000..00ee6f8 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/automat/__pycache__/_visualize.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/automat/_core.py b/WebCrawler/venv/Lib/site-packages/automat/_core.py new file mode 100644 index 0000000..4118a4b --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/automat/_core.py @@ -0,0 +1,165 @@ +# -*- test-case-name: automat._test.test_core -*- + +""" +A core state-machine abstraction. + +Perhaps something that could be replaced with or integrated into machinist. +""" + +from itertools import chain + +_NO_STATE = "" + + +class NoTransition(Exception): + """ + A finite state machine in C{state} has no transition for C{symbol}. + + @param state: the finite state machine's state at the time of the + illegal transition. + + @param symbol: the input symbol for which no transition exists. + """ + + def __init__(self, state, symbol): + self.state = state + self.symbol = symbol + super(Exception, self).__init__( + "no transition for {} in {}".format(symbol, state) + ) + + +class Automaton(object): + """ + A declaration of a finite state machine. + + Note that this is not the machine itself; it is immutable. + """ + + def __init__(self): + """ + Initialize the set of transitions and the initial state. + """ + self._initialState = _NO_STATE + self._transitions = set() + + + @property + def initialState(self): + """ + Return this automaton's initial state. + """ + return self._initialState + + + @initialState.setter + def initialState(self, state): + """ + Set this automaton's initial state. Raises a ValueError if + this automaton already has an initial state. + """ + + if self._initialState is not _NO_STATE: + raise ValueError( + "initial state already set to {}".format(self._initialState)) + + self._initialState = state + + + def addTransition(self, inState, inputSymbol, outState, outputSymbols): + """ + Add the given transition to the outputSymbol. Raise ValueError if + there is already a transition with the same inState and inputSymbol. + """ + # keeping self._transitions in a flat list makes addTransition + # O(n^2), but state machines don't tend to have hundreds of + # transitions. + for (anInState, anInputSymbol, anOutState, _) in self._transitions: + if (anInState == inState and anInputSymbol == inputSymbol): + raise ValueError( + "already have transition from {} via {}".format(inState, inputSymbol)) + self._transitions.add( + (inState, inputSymbol, outState, tuple(outputSymbols)) + ) + + + def allTransitions(self): + """ + All transitions. + """ + return frozenset(self._transitions) + + + def inputAlphabet(self): + """ + The full set of symbols acceptable to this automaton. + """ + return {inputSymbol for (inState, inputSymbol, outState, + outputSymbol) in self._transitions} + + + def outputAlphabet(self): + """ + The full set of symbols which can be produced by this automaton. + """ + return set( + chain.from_iterable( + outputSymbols for + (inState, inputSymbol, outState, outputSymbols) + in self._transitions + ) + ) + + + def states(self): + """ + All valid states; "Q" in the mathematical description of a state + machine. + """ + return frozenset( + chain.from_iterable( + (inState, outState) + for + (inState, inputSymbol, outState, outputSymbol) + in self._transitions + ) + ) + + + def outputForInput(self, inState, inputSymbol): + """ + A 2-tuple of (outState, outputSymbols) for inputSymbol. + """ + for (anInState, anInputSymbol, + outState, outputSymbols) in self._transitions: + if (inState, inputSymbol) == (anInState, anInputSymbol): + return (outState, list(outputSymbols)) + raise NoTransition(state=inState, symbol=inputSymbol) + + +class Transitioner(object): + """ + The combination of a current state and an L{Automaton}. + """ + + def __init__(self, automaton, initialState): + self._automaton = automaton + self._state = initialState + self._tracer = None + + def setTrace(self, tracer): + self._tracer = tracer + + def transition(self, inputSymbol): + """ + Transition between states, returning any outputs. + """ + outState, outputSymbols = self._automaton.outputForInput(self._state, + inputSymbol) + outTracer = None + if self._tracer: + outTracer = self._tracer(self._state._name(), + inputSymbol._name(), + outState._name()) + self._state = outState + return (outputSymbols, outTracer) diff --git a/WebCrawler/venv/Lib/site-packages/automat/_discover.py b/WebCrawler/venv/Lib/site-packages/automat/_discover.py new file mode 100644 index 0000000..c0d88ba --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/automat/_discover.py @@ -0,0 +1,144 @@ +import collections +import inspect +from automat import MethodicalMachine +from twisted.python.modules import PythonModule, getModule + + +def isOriginalLocation(attr): + """ + Attempt to discover if this appearance of a PythonAttribute + representing a class refers to the module where that class was + defined. + """ + sourceModule = inspect.getmodule(attr.load()) + if sourceModule is None: + return False + + currentModule = attr + while not isinstance(currentModule, PythonModule): + currentModule = currentModule.onObject + + return currentModule.name == sourceModule.__name__ + + +def findMachinesViaWrapper(within): + """ + Recursively yield L{MethodicalMachine}s and their FQPNs within a + L{PythonModule} or a L{twisted.python.modules.PythonAttribute} + wrapper object. + + Note that L{PythonModule}s may refer to packages, as well. + + The discovery heuristic considers L{MethodicalMachine} instances + that are module-level attributes or class-level attributes + accessible from module scope. Machines inside nested classes will + be discovered, but those returned from functions or methods will not be. + + @type within: L{PythonModule} or L{twisted.python.modules.PythonAttribute} + @param within: Where to start the search. + + @return: a generator which yields FQPN, L{MethodicalMachine} pairs. + """ + queue = collections.deque([within]) + visited = set() + + while queue: + attr = queue.pop() + value = attr.load() + + if isinstance(value, MethodicalMachine) and value not in visited: + visited.add(value) + yield attr.name, value + elif (inspect.isclass(value) and isOriginalLocation(attr) and + value not in visited): + visited.add(value) + queue.extendleft(attr.iterAttributes()) + elif isinstance(attr, PythonModule) and value not in visited: + visited.add(value) + queue.extendleft(attr.iterAttributes()) + queue.extendleft(attr.iterModules()) + + +class InvalidFQPN(Exception): + """ + The given FQPN was not a dot-separated list of Python objects. + """ + + +class NoModule(InvalidFQPN): + """ + A prefix of the FQPN was not an importable module or package. + """ + + +class NoObject(InvalidFQPN): + """ + A suffix of the FQPN was not an accessible object + """ + + +def wrapFQPN(fqpn): + """ + Given an FQPN, retrieve the object via the global Python module + namespace and wrap it with a L{PythonModule} or a + L{twisted.python.modules.PythonAttribute}. + """ + # largely cribbed from t.p.reflect.namedAny + + if not fqpn: + raise InvalidFQPN("FQPN was empty") + + components = collections.deque(fqpn.split('.')) + + if '' in components: + raise InvalidFQPN( + "name must be a string giving a '.'-separated list of Python " + "identifiers, not %r" % (fqpn,)) + + component = components.popleft() + try: + module = getModule(component) + except KeyError: + raise NoModule(component) + + # find the bottom-most module + while components: + component = components.popleft() + try: + module = module[component] + except KeyError: + components.appendleft(component) + break + else: + module.load() + else: + return module + + # find the bottom-most attribute + attribute = module + for component in components: + try: + attribute = next(child for child in attribute.iterAttributes() + if child.name.rsplit('.', 1)[-1] == component) + except StopIteration: + raise NoObject('{}.{}'.format(attribute.name, component)) + + return attribute + + +def findMachines(fqpn): + """ + Recursively yield L{MethodicalMachine}s and their FQPNs in and + under the a Python object specified by an FQPN. + + The discovery heuristic considers L{MethodicalMachine} instances + that are module-level attributes or class-level attributes + accessible from module scope. Machines inside nested classes will + be discovered, but those returned from functions or methods will not be. + + @type within: an FQPN + @param within: Where to start the search. + + @return: a generator which yields FQPN, L{MethodicalMachine} pairs. + """ + return findMachinesViaWrapper(wrapFQPN(fqpn)) diff --git a/WebCrawler/venv/Lib/site-packages/automat/_introspection.py b/WebCrawler/venv/Lib/site-packages/automat/_introspection.py new file mode 100644 index 0000000..3f7307d --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/automat/_introspection.py @@ -0,0 +1,45 @@ +""" +Python introspection helpers. +""" + +from types import CodeType as code, FunctionType as function + + +def copycode(template, changes): + names = [ + "argcount", "nlocals", "stacksize", "flags", "code", "consts", + "names", "varnames", "filename", "name", "firstlineno", "lnotab", + "freevars", "cellvars" + ] + if hasattr(code, "co_kwonlyargcount"): + names.insert(1, "kwonlyargcount") + if hasattr(code, "co_posonlyargcount"): + # PEP 570 added "positional only arguments" + names.insert(1, "posonlyargcount") + values = [ + changes.get(name, getattr(template, "co_" + name)) + for name in names + ] + return code(*values) + + + +def copyfunction(template, funcchanges, codechanges): + names = [ + "globals", "name", "defaults", "closure", + ] + values = [ + funcchanges.get(name, getattr(template, "__" + name + "__")) + for name in names + ] + return function(copycode(template.__code__, codechanges), *values) + + +def preserveName(f): + """ + Preserve the name of the given function on the decorated function. + """ + def decorator(decorated): + return copyfunction(decorated, + dict(name=f.__name__), dict(name=f.__name__)) + return decorator diff --git a/WebCrawler/venv/Lib/site-packages/automat/_methodical.py b/WebCrawler/venv/Lib/site-packages/automat/_methodical.py new file mode 100644 index 0000000..84fcd36 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/automat/_methodical.py @@ -0,0 +1,474 @@ +# -*- test-case-name: automat._test.test_methodical -*- + +import collections +from functools import wraps +from itertools import count + +try: + # Python 3 + from inspect import getfullargspec as getArgsSpec +except ImportError: + # Python 2 + from inspect import getargspec as getArgsSpec + +import attr +import six + +from ._core import Transitioner, Automaton +from ._introspection import preserveName + + +ArgSpec = collections.namedtuple('ArgSpec', ['args', 'varargs', 'varkw', + 'defaults', 'kwonlyargs', + 'kwonlydefaults', 'annotations']) + + +def _getArgSpec(func): + """ + Normalize inspect.ArgSpec across python versions + and convert mutable attributes to immutable types. + + :param Callable func: A function. + :return: The function's ArgSpec. + :rtype: ArgSpec + """ + spec = getArgsSpec(func) + return ArgSpec( + args=tuple(spec.args), + varargs=spec.varargs, + varkw=spec.varkw if six.PY3 else spec.keywords, + defaults=spec.defaults if spec.defaults else (), + kwonlyargs=tuple(spec.kwonlyargs) if six.PY3 else (), + kwonlydefaults=( + tuple(spec.kwonlydefaults.items()) + if spec.kwonlydefaults else () + ) if six.PY3 else (), + annotations=tuple(spec.annotations.items()) if six.PY3 else (), + ) + + +def _getArgNames(spec): + """ + Get the name of all arguments defined in a function signature. + + The name of * and ** arguments is normalized to "*args" and "**kwargs". + + :param ArgSpec spec: A function to interrogate for a signature. + :return: The set of all argument names in `func`s signature. + :rtype: Set[str] + """ + return set( + spec.args + + spec.kwonlyargs + + (('*args',) if spec.varargs else ()) + + (('**kwargs',) if spec.varkw else ()) + + spec.annotations + ) + + +def _keywords_only(f): + """ + Decorate a function so all its arguments must be passed by keyword. + + A useful utility for decorators that take arguments so that they don't + accidentally get passed the thing they're decorating as their first + argument. + + Only works for methods right now. + """ + @wraps(f) + def g(self, **kw): + return f(self, **kw) + return g + + +@attr.s(frozen=True) +class MethodicalState(object): + """ + A state for a L{MethodicalMachine}. + """ + machine = attr.ib(repr=False) + method = attr.ib() + serialized = attr.ib(repr=False) + + def upon(self, input, enter, outputs, collector=list): + """ + Declare a state transition within the :class:`automat.MethodicalMachine` + associated with this :class:`automat.MethodicalState`: + upon the receipt of the `input`, enter the `state`, + emitting each output in `outputs`. + + :param MethodicalInput input: The input triggering a state transition. + :param MethodicalState enter: The resulting state. + :param Iterable[MethodicalOutput] outputs: The outputs to be triggered + as a result of the declared state transition. + :param Callable collector: The function to be used when collecting + output return values. + + :raises TypeError: if any of the `outputs` signatures do not match + the `inputs` signature. + :raises ValueError: if the state transition from `self` via `input` + has already been defined. + """ + inputArgs = _getArgNames(input.argSpec) + for output in outputs: + outputArgs = _getArgNames(output.argSpec) + if not outputArgs.issubset(inputArgs): + raise TypeError( + "method {input} signature {inputSignature} " + "does not match output {output} " + "signature {outputSignature}".format( + input=input.method.__name__, + output=output.method.__name__, + inputSignature=getArgsSpec(input.method), + outputSignature=getArgsSpec(output.method), + )) + self.machine._oneTransition(self, input, enter, outputs, collector) + + def _name(self): + return self.method.__name__ + + +def _transitionerFromInstance(oself, symbol, automaton): + """ + Get a L{Transitioner} + """ + transitioner = getattr(oself, symbol, None) + if transitioner is None: + transitioner = Transitioner( + automaton, + automaton.initialState, + ) + setattr(oself, symbol, transitioner) + return transitioner + + +def _empty(): + pass + +def _docstring(): + """docstring""" + +def assertNoCode(inst, attribute, f): + # The function body must be empty, i.e. "pass" or "return None", which + # both yield the same bytecode: LOAD_CONST (None), RETURN_VALUE. We also + # accept functions with only a docstring, which yields slightly different + # bytecode, because the "None" is put in a different constant slot. + + # Unfortunately, this does not catch function bodies that return a + # constant value, e.g. "return 1", because their code is identical to a + # "return None". They differ in the contents of their constant table, but + # checking that would require us to parse the bytecode, find the index + # being returned, then making sure the table has a None at that index. + + if f.__code__.co_code not in (_empty.__code__.co_code, + _docstring.__code__.co_code): + raise ValueError("function body must be empty") + + +def _filterArgs(args, kwargs, inputSpec, outputSpec): + """ + Filter out arguments that were passed to input that output won't accept. + + :param tuple args: The *args that input received. + :param dict kwargs: The **kwargs that input received. + :param ArgSpec inputSpec: The input's arg spec. + :param ArgSpec outputSpec: The output's arg spec. + :return: The args and kwargs that output will accept. + :rtype: Tuple[tuple, dict] + """ + named_args = tuple(zip(inputSpec.args[1:], args)) + if outputSpec.varargs: + # Only return all args if the output accepts *args. + return_args = args + else: + # Filter out arguments that don't appear + # in the output's method signature. + return_args = [v for n, v in named_args if n in outputSpec.args] + + # Get any of input's default arguments that were not passed. + passed_arg_names = tuple(kwargs) + for name, value in named_args: + passed_arg_names += (name, value) + defaults = zip(inputSpec.args[::-1], inputSpec.defaults[::-1]) + full_kwargs = {n: v for n, v in defaults if n not in passed_arg_names} + full_kwargs.update(kwargs) + + if outputSpec.varkw: + # Only pass all kwargs if the output method accepts **kwargs. + return_kwargs = full_kwargs + else: + # Filter out names that the output method does not accept. + all_accepted_names = outputSpec.args[1:] + outputSpec.kwonlyargs + return_kwargs = {n: v for n, v in full_kwargs.items() + if n in all_accepted_names} + + return return_args, return_kwargs + + +@attr.s(eq=False, hash=False) +class MethodicalInput(object): + """ + An input for a L{MethodicalMachine}. + """ + automaton = attr.ib(repr=False) + method = attr.ib(validator=assertNoCode) + symbol = attr.ib(repr=False) + collectors = attr.ib(default=attr.Factory(dict), repr=False) + argSpec = attr.ib(init=False, repr=False) + + @argSpec.default + def _buildArgSpec(self): + return _getArgSpec(self.method) + + def __get__(self, oself, type=None): + """ + Return a function that takes no arguments and returns values returned + by output functions produced by the given L{MethodicalInput} in + C{oself}'s current state. + """ + transitioner = _transitionerFromInstance(oself, self.symbol, + self.automaton) + @preserveName(self.method) + @wraps(self.method) + def doInput(*args, **kwargs): + self.method(oself, *args, **kwargs) + previousState = transitioner._state + (outputs, outTracer) = transitioner.transition(self) + collector = self.collectors[previousState] + values = [] + for output in outputs: + if outTracer: + outTracer(output._name()) + a, k = _filterArgs(args, kwargs, self.argSpec, output.argSpec) + value = output(oself, *a, **k) + values.append(value) + return collector(values) + return doInput + + def _name(self): + return self.method.__name__ + + +@attr.s(frozen=True) +class MethodicalOutput(object): + """ + An output for a L{MethodicalMachine}. + """ + machine = attr.ib(repr=False) + method = attr.ib() + argSpec = attr.ib(init=False, repr=False) + + @argSpec.default + def _buildArgSpec(self): + return _getArgSpec(self.method) + + def __get__(self, oself, type=None): + """ + Outputs are private, so raise an exception when we attempt to get one. + """ + raise AttributeError( + "{cls}.{method} is a state-machine output method; " + "to produce this output, call an input method instead.".format( + cls=type.__name__, + method=self.method.__name__ + ) + ) + + + def __call__(self, oself, *args, **kwargs): + """ + Call the underlying method. + """ + return self.method(oself, *args, **kwargs) + + def _name(self): + return self.method.__name__ + +@attr.s(eq=False, hash=False) +class MethodicalTracer(object): + automaton = attr.ib(repr=False) + symbol = attr.ib(repr=False) + + + def __get__(self, oself, type=None): + transitioner = _transitionerFromInstance(oself, self.symbol, + self.automaton) + def setTrace(tracer): + transitioner.setTrace(tracer) + return setTrace + + + +counter = count() +def gensym(): + """ + Create a unique Python identifier. + """ + return "_symbol_" + str(next(counter)) + + + +class MethodicalMachine(object): + """ + A :class:`MethodicalMachine` is an interface to an `Automaton` + that uses methods on a class. + """ + + def __init__(self): + self._automaton = Automaton() + self._reducers = {} + self._symbol = gensym() + + + def __get__(self, oself, type=None): + """ + L{MethodicalMachine} is an implementation detail for setting up + class-level state; applications should never need to access it on an + instance. + """ + if oself is not None: + raise AttributeError( + "MethodicalMachine is an implementation detail.") + return self + + + @_keywords_only + def state(self, initial=False, terminal=False, + serialized=None): + """ + Declare a state, possibly an initial state or a terminal state. + + This is a decorator for methods, but it will modify the method so as + not to be callable any more. + + :param bool initial: is this state the initial state? + Only one state on this :class:`automat.MethodicalMachine` + may be an initial state; more than one is an error. + + :param bool terminal: Is this state a terminal state? + i.e. a state that the machine can end up in? + (This is purely informational at this point.) + + :param Hashable serialized: a serializable value + to be used to represent this state to external systems. + This value should be hashable; + :py:func:`unicode` is a good type to use. + """ + def decorator(stateMethod): + state = MethodicalState(machine=self, + method=stateMethod, + serialized=serialized) + if initial: + self._automaton.initialState = state + return state + return decorator + + + @_keywords_only + def input(self): + """ + Declare an input. + + This is a decorator for methods. + """ + def decorator(inputMethod): + return MethodicalInput(automaton=self._automaton, + method=inputMethod, + symbol=self._symbol) + return decorator + + + @_keywords_only + def output(self): + """ + Declare an output. + + This is a decorator for methods. + + This method will be called when the state machine transitions to this + state as specified in the decorated `output` method. + """ + def decorator(outputMethod): + return MethodicalOutput(machine=self, method=outputMethod) + return decorator + + + def _oneTransition(self, startState, inputToken, endState, outputTokens, + collector): + """ + See L{MethodicalState.upon}. + """ + # FIXME: tests for all of this (some of it is wrong) + # if not isinstance(startState, MethodicalState): + # raise NotImplementedError("start state {} isn't a state" + # .format(startState)) + # if not isinstance(inputToken, MethodicalInput): + # raise NotImplementedError("start state {} isn't an input" + # .format(inputToken)) + # if not isinstance(endState, MethodicalState): + # raise NotImplementedError("end state {} isn't a state" + # .format(startState)) + # for output in outputTokens: + # if not isinstance(endState, MethodicalState): + # raise NotImplementedError("output state {} isn't a state" + # .format(endState)) + self._automaton.addTransition(startState, inputToken, endState, + tuple(outputTokens)) + inputToken.collectors[startState] = collector + + + @_keywords_only + def serializer(self): + """ + + """ + def decorator(decoratee): + @wraps(decoratee) + def serialize(oself): + transitioner = _transitionerFromInstance(oself, self._symbol, + self._automaton) + return decoratee(oself, transitioner._state.serialized) + return serialize + return decorator + + @_keywords_only + def unserializer(self): + """ + + """ + def decorator(decoratee): + @wraps(decoratee) + def unserialize(oself, *args, **kwargs): + state = decoratee(oself, *args, **kwargs) + mapping = {} + for eachState in self._automaton.states(): + mapping[eachState.serialized] = eachState + transitioner = _transitionerFromInstance( + oself, self._symbol, self._automaton) + transitioner._state = mapping[state] + return None # it's on purpose + return unserialize + return decorator + + @property + def _setTrace(self): + return MethodicalTracer(self._automaton, self._symbol) + + def asDigraph(self): + """ + Generate a L{graphviz.Digraph} that represents this machine's + states and transitions. + + @return: L{graphviz.Digraph} object; for more information, please + see the documentation for + U{graphviz} + + """ + from ._visualize import makeDigraph + return makeDigraph( + self._automaton, + stateAsString=lambda state: state.method.__name__, + inputAsString=lambda input: input.method.__name__, + outputAsString=lambda output: output.method.__name__, + ) diff --git a/WebCrawler/venv/Lib/site-packages/automat/_test/__init__.py b/WebCrawler/venv/Lib/site-packages/automat/_test/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/WebCrawler/venv/Lib/site-packages/automat/_test/__pycache__/__init__.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/automat/_test/__pycache__/__init__.cpython-39.pyc new file mode 100644 index 0000000..2890609 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/automat/_test/__pycache__/__init__.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/automat/_test/__pycache__/test_core.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/automat/_test/__pycache__/test_core.cpython-39.pyc new file mode 100644 index 0000000..41e6357 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/automat/_test/__pycache__/test_core.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/automat/_test/__pycache__/test_discover.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/automat/_test/__pycache__/test_discover.cpython-39.pyc new file mode 100644 index 0000000..25dd410 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/automat/_test/__pycache__/test_discover.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/automat/_test/__pycache__/test_methodical.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/automat/_test/__pycache__/test_methodical.cpython-39.pyc new file mode 100644 index 0000000..088165a Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/automat/_test/__pycache__/test_methodical.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/automat/_test/__pycache__/test_trace.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/automat/_test/__pycache__/test_trace.cpython-39.pyc new file mode 100644 index 0000000..9a206e9 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/automat/_test/__pycache__/test_trace.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/automat/_test/__pycache__/test_visualize.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/automat/_test/__pycache__/test_visualize.cpython-39.pyc new file mode 100644 index 0000000..a4526c0 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/automat/_test/__pycache__/test_visualize.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/automat/_test/test_core.py b/WebCrawler/venv/Lib/site-packages/automat/_test/test_core.py new file mode 100644 index 0000000..a9ff050 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/automat/_test/test_core.py @@ -0,0 +1,86 @@ + +from .._core import Automaton, NoTransition + +from unittest import TestCase + +class CoreTests(TestCase): + """ + Tests for Automat's (currently private, implementation detail) core. + """ + + def test_NoTransition(self): + """ + A L{NoTransition} exception describes the state and input symbol + that caused it. + """ + # NoTransition requires two arguments + with self.assertRaises(TypeError): + NoTransition() + + state = "current-state" + symbol = "transitionless-symbol" + noTransitionException = NoTransition(state=state, symbol=symbol) + + self.assertIs(noTransitionException.symbol, symbol) + + self.assertIn(state, str(noTransitionException)) + self.assertIn(symbol, str(noTransitionException)) + + + def test_noOutputForInput(self): + """ + L{Automaton.outputForInput} raises L{NoTransition} if no + transition for that input is defined. + """ + a = Automaton() + self.assertRaises(NoTransition, a.outputForInput, + "no-state", "no-symbol") + + + def test_oneTransition(self): + """ + L{Automaton.addTransition} adds its input symbol to + L{Automaton.inputAlphabet}, all its outputs to + L{Automaton.outputAlphabet}, and causes L{Automaton.outputForInput} to + start returning the new state and output symbols. + """ + a = Automaton() + a.addTransition("beginning", "begin", "ending", ["end"]) + self.assertEqual(a.inputAlphabet(), {"begin"}) + self.assertEqual(a.outputAlphabet(), {"end"}) + self.assertEqual(a.outputForInput("beginning", "begin"), + ("ending", ["end"])) + self.assertEqual(a.states(), {"beginning", "ending"}) + + + def test_oneTransition_nonIterableOutputs(self): + """ + L{Automaton.addTransition} raises a TypeError when given outputs + that aren't iterable and doesn't add any transitions. + """ + a = Automaton() + nonIterableOutputs = 1 + self.assertRaises( + TypeError, + a.addTransition, + "fromState", "viaSymbol", "toState", nonIterableOutputs) + self.assertFalse(a.inputAlphabet()) + self.assertFalse(a.outputAlphabet()) + self.assertFalse(a.states()) + self.assertFalse(a.allTransitions()) + + + def test_initialState(self): + """ + L{Automaton.initialState} is a descriptor that sets the initial + state if it's not yet set, and raises L{ValueError} if it is. + + """ + a = Automaton() + a.initialState = "a state" + self.assertEqual(a.initialState, "a state") + with self.assertRaises(ValueError): + a.initialState = "another state" + + +# FIXME: addTransition for transition that's been added before diff --git a/WebCrawler/venv/Lib/site-packages/automat/_test/test_discover.py b/WebCrawler/venv/Lib/site-packages/automat/_test/test_discover.py new file mode 100644 index 0000000..2aaf16a --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/automat/_test/test_discover.py @@ -0,0 +1,609 @@ +import operator +import os +import shutil +import sys +import textwrap +import tempfile +from unittest import skipIf, TestCase + +import six + + +def isTwistedInstalled(): + try: + __import__('twisted') + except ImportError: + return False + else: + return True + + +class _WritesPythonModules(TestCase): + """ + A helper that enables generating Python module test fixtures. + """ + + def setUp(self): + super(_WritesPythonModules, self).setUp() + + from twisted.python.modules import getModule, PythonPath + from twisted.python.filepath import FilePath + + self.getModule = getModule + self.PythonPath = PythonPath + self.FilePath = FilePath + + self.originalSysModules = set(sys.modules.keys()) + self.savedSysPath = sys.path[:] + + self.pathDir = tempfile.mkdtemp() + self.makeImportable(self.pathDir) + + def tearDown(self): + super(_WritesPythonModules, self).tearDown() + + sys.path[:] = self.savedSysPath + modulesToDelete = six.viewkeys(sys.modules) - self.originalSysModules + for module in modulesToDelete: + del sys.modules[module] + + shutil.rmtree(self.pathDir) + + def makeImportable(self, path): + sys.path.append(path) + + def writeSourceInto(self, source, path, moduleName): + directory = self.FilePath(path) + + module = directory.child(moduleName) + # FilePath always opens a file in binary mode - but that will + # break on Python 3 + with open(module.path, 'w') as f: + f.write(textwrap.dedent(source)) + + return self.PythonPath([directory.path]) + + def makeModule(self, source, path, moduleName): + pythonModuleName, _ = os.path.splitext(moduleName) + return self.writeSourceInto(source, path, moduleName)[pythonModuleName] + + def attributesAsDict(self, hasIterAttributes): + return {attr.name: attr for attr in hasIterAttributes.iterAttributes()} + + def loadModuleAsDict(self, module): + module.load() + return self.attributesAsDict(module) + + def makeModuleAsDict(self, source, path, name): + return self.loadModuleAsDict(self.makeModule(source, path, name)) + + +@skipIf(not isTwistedInstalled(), "Twisted is not installed.") +class OriginalLocationTests(_WritesPythonModules): + """ + Tests that L{isOriginalLocation} detects when a + L{PythonAttribute}'s FQPN refers to an object inside the module + where it was defined. + + For example: A L{twisted.python.modules.PythonAttribute} with a + name of 'foo.bar' that refers to a 'bar' object defined in module + 'baz' does *not* refer to bar's original location, while a + L{PythonAttribute} with a name of 'baz.bar' does. + + """ + def setUp(self): + super(OriginalLocationTests, self).setUp() + from .._discover import isOriginalLocation + self.isOriginalLocation = isOriginalLocation + + def test_failsWithNoModule(self): + """ + L{isOriginalLocation} returns False when the attribute refers to an + object whose source module cannot be determined. + """ + source = """\ + class Fake(object): + pass + hasEmptyModule = Fake() + hasEmptyModule.__module__ = None + """ + + moduleDict = self.makeModuleAsDict(source, + self.pathDir, + 'empty_module_attr.py') + + self.assertFalse(self.isOriginalLocation( + moduleDict['empty_module_attr.hasEmptyModule'])) + + def test_failsWithDifferentModule(self): + """ + L{isOriginalLocation} returns False when the attribute refers to + an object outside of the module where that object was defined. + """ + originalSource = """\ + class ImportThisClass(object): + pass + importThisObject = ImportThisClass() + importThisNestingObject = ImportThisClass() + importThisNestingObject.nestedObject = ImportThisClass() + """ + + importingSource = """\ + from original import (ImportThisClass, + importThisObject, + importThisNestingObject) + """ + + self.makeModule(originalSource, self.pathDir, 'original.py') + importingDict = self.makeModuleAsDict(importingSource, + self.pathDir, + 'importing.py') + self.assertFalse( + self.isOriginalLocation( + importingDict['importing.ImportThisClass'])) + self.assertFalse( + self.isOriginalLocation( + importingDict['importing.importThisObject'])) + + nestingObject = importingDict['importing.importThisNestingObject'] + nestingObjectDict = self.attributesAsDict(nestingObject) + nestedObject = nestingObjectDict[ + 'importing.importThisNestingObject.nestedObject'] + + self.assertFalse(self.isOriginalLocation(nestedObject)) + + def test_succeedsWithSameModule(self): + """ + L{isOriginalLocation} returns True when the attribute refers to an + object inside the module where that object was defined. + """ + mSource = textwrap.dedent(""" + class ThisClassWasDefinedHere(object): + pass + anObject = ThisClassWasDefinedHere() + aNestingObject = ThisClassWasDefinedHere() + aNestingObject.nestedObject = ThisClassWasDefinedHere() + """) + mDict = self.makeModuleAsDict(mSource, self.pathDir, 'm.py') + self.assertTrue(self.isOriginalLocation( + mDict['m.ThisClassWasDefinedHere'])) + self.assertTrue(self.isOriginalLocation(mDict['m.aNestingObject'])) + + nestingObject = mDict['m.aNestingObject'] + nestingObjectDict = self.attributesAsDict(nestingObject) + nestedObject = nestingObjectDict['m.aNestingObject.nestedObject'] + + self.assertTrue(self.isOriginalLocation(nestedObject)) + + +@skipIf(not isTwistedInstalled(), "Twisted is not installed.") +class FindMachinesViaWrapperTests(_WritesPythonModules): + """ + L{findMachinesViaWrapper} recursively yields FQPN, + L{MethodicalMachine} pairs in and under a given + L{twisted.python.modules.PythonModule} or + L{twisted.python.modules.PythonAttribute}. + """ + TEST_MODULE_SOURCE = """ + from automat import MethodicalMachine + + + class PythonClass(object): + _classMachine = MethodicalMachine() + + class NestedClass(object): + _nestedClassMachine = MethodicalMachine() + + ignoredAttribute = "I am ignored." + + def ignoredMethod(self): + "I am also ignored." + + rootLevelMachine = MethodicalMachine() + ignoredPythonObject = PythonClass() + anotherIgnoredPythonObject = "I am ignored." + """ + + def setUp(self): + super(FindMachinesViaWrapperTests, self).setUp() + from .._discover import findMachinesViaWrapper + self.findMachinesViaWrapper = findMachinesViaWrapper + + def test_yieldsMachine(self): + """ + When given a L{twisted.python.modules.PythonAttribute} that refers + directly to a L{MethodicalMachine}, L{findMachinesViaWrapper} + yields that machine and its FQPN. + """ + source = """\ + from automat import MethodicalMachine + + rootMachine = MethodicalMachine() + """ + + moduleDict = self.makeModuleAsDict(source, self.pathDir, 'root.py') + rootMachine = moduleDict['root.rootMachine'] + self.assertIn(('root.rootMachine', rootMachine.load()), + list(self.findMachinesViaWrapper(rootMachine))) + + def test_yieldsMachineInClass(self): + """ + When given a L{twisted.python.modules.PythonAttribute} that refers + to a class that contains a L{MethodicalMachine} as a class + variable, L{findMachinesViaWrapper} yields that machine and + its FQPN. + """ + source = """\ + from automat import MethodicalMachine + + class PythonClass(object): + _classMachine = MethodicalMachine() + """ + moduleDict = self.makeModuleAsDict(source, self.pathDir, 'clsmod.py') + PythonClass = moduleDict['clsmod.PythonClass'] + self.assertIn(('clsmod.PythonClass._classMachine', + PythonClass.load()._classMachine), + list(self.findMachinesViaWrapper(PythonClass))) + + def test_yieldsMachineInNestedClass(self): + """ + When given a L{twisted.python.modules.PythonAttribute} that refers + to a nested class that contains a L{MethodicalMachine} as a + class variable, L{findMachinesViaWrapper} yields that machine + and its FQPN. + """ + source = """\ + from automat import MethodicalMachine + + class PythonClass(object): + class NestedClass(object): + _classMachine = MethodicalMachine() + """ + moduleDict = self.makeModuleAsDict(source, + self.pathDir, + 'nestedcls.py') + + PythonClass = moduleDict['nestedcls.PythonClass'] + self.assertIn(('nestedcls.PythonClass.NestedClass._classMachine', + PythonClass.load().NestedClass._classMachine), + list(self.findMachinesViaWrapper(PythonClass))) + + def test_yieldsMachineInModule(self): + """ + When given a L{twisted.python.modules.PythonModule} that refers to + a module that contains a L{MethodicalMachine}, + L{findMachinesViaWrapper} yields that machine and its FQPN. + """ + source = """\ + from automat import MethodicalMachine + + rootMachine = MethodicalMachine() + """ + module = self.makeModule(source, self.pathDir, 'root.py') + rootMachine = self.loadModuleAsDict(module)['root.rootMachine'].load() + self.assertIn(('root.rootMachine', rootMachine), + list(self.findMachinesViaWrapper(module))) + + def test_yieldsMachineInClassInModule(self): + """ + When given a L{twisted.python.modules.PythonModule} that refers to + the original module of a class containing a + L{MethodicalMachine}, L{findMachinesViaWrapper} yields that + machine and its FQPN. + """ + source = """\ + from automat import MethodicalMachine + + class PythonClass(object): + _classMachine = MethodicalMachine() + """ + module = self.makeModule(source, self.pathDir, 'clsmod.py') + PythonClass = self.loadModuleAsDict( + module)['clsmod.PythonClass'].load() + self.assertIn(('clsmod.PythonClass._classMachine', + PythonClass._classMachine), + list(self.findMachinesViaWrapper(module))) + + def test_yieldsMachineInNestedClassInModule(self): + """ + When given a L{twisted.python.modules.PythonModule} that refers to + the original module of a nested class containing a + L{MethodicalMachine}, L{findMachinesViaWrapper} yields that + machine and its FQPN. + """ + source = """\ + from automat import MethodicalMachine + + class PythonClass(object): + class NestedClass(object): + _classMachine = MethodicalMachine() + """ + module = self.makeModule(source, self.pathDir, 'nestedcls.py') + PythonClass = self.loadModuleAsDict( + module)['nestedcls.PythonClass'].load() + + self.assertIn(('nestedcls.PythonClass.NestedClass._classMachine', + PythonClass.NestedClass._classMachine), + list(self.findMachinesViaWrapper(module))) + + def test_ignoresImportedClass(self): + """ + When given a L{twisted.python.modules.PythonAttribute} that refers + to a class imported from another module, any + L{MethodicalMachine}s on that class are ignored. + + This behavior ensures that a machine is only discovered on a + class when visiting the module where that class was defined. + """ + originalSource = """ + from automat import MethodicalMachine + + class PythonClass(object): + _classMachine = MethodicalMachine() + """ + + importingSource = """ + from original import PythonClass + """ + + self.makeModule(originalSource, self.pathDir, 'original.py') + importingModule = self.makeModule(importingSource, + self.pathDir, + 'importing.py') + + self.assertFalse(list(self.findMachinesViaWrapper(importingModule))) + + def test_descendsIntoPackages(self): + """ + L{findMachinesViaWrapper} descends into packages to discover + machines. + """ + pythonPath = self.PythonPath([self.pathDir]) + package = self.FilePath(self.pathDir).child("test_package") + package.makedirs() + package.child('__init__.py').touch() + + source = """ + from automat import MethodicalMachine + + + class PythonClass(object): + _classMachine = MethodicalMachine() + + + rootMachine = MethodicalMachine() + """ + self.makeModule(source, package.path, 'module.py') + + test_package = pythonPath['test_package'] + machines = sorted(self.findMachinesViaWrapper(test_package), + key=operator.itemgetter(0)) + + moduleDict = self.loadModuleAsDict(test_package['module']) + rootMachine = moduleDict['test_package.module.rootMachine'].load() + PythonClass = moduleDict['test_package.module.PythonClass'].load() + + expectedMachines = sorted( + [('test_package.module.rootMachine', + rootMachine), + ('test_package.module.PythonClass._classMachine', + PythonClass._classMachine)], key=operator.itemgetter(0)) + + self.assertEqual(expectedMachines, machines) + + def test_infiniteLoop(self): + """ + L{findMachinesViaWrapper} ignores infinite loops. + + Note this test can't fail - it can only run forever! + """ + source = """ + class InfiniteLoop(object): + pass + + InfiniteLoop.loop = InfiniteLoop + """ + module = self.makeModule(source, self.pathDir, 'loop.py') + self.assertFalse(list(self.findMachinesViaWrapper(module))) + + +@skipIf(not isTwistedInstalled(), "Twisted is not installed.") +class WrapFQPNTests(TestCase): + """ + Tests that ensure L{wrapFQPN} loads the + L{twisted.python.modules.PythonModule} or + L{twisted.python.modules.PythonAttribute} for a given FQPN. + """ + + def setUp(self): + from twisted.python.modules import PythonModule, PythonAttribute + from .._discover import wrapFQPN, InvalidFQPN, NoModule, NoObject + + self.PythonModule = PythonModule + self.PythonAttribute = PythonAttribute + self.wrapFQPN = wrapFQPN + self.InvalidFQPN = InvalidFQPN + self.NoModule = NoModule + self.NoObject = NoObject + + def assertModuleWrapperRefersTo(self, moduleWrapper, module): + """ + Assert that a L{twisted.python.modules.PythonModule} refers to a + particular Python module. + """ + self.assertIsInstance(moduleWrapper, self.PythonModule) + self.assertEqual(moduleWrapper.name, module.__name__) + self.assertIs(moduleWrapper.load(), module) + + def assertAttributeWrapperRefersTo(self, attributeWrapper, fqpn, obj): + """ + Assert that a L{twisted.python.modules.PythonAttribute} refers to a + particular Python object. + """ + self.assertIsInstance(attributeWrapper, self.PythonAttribute) + self.assertEqual(attributeWrapper.name, fqpn) + self.assertIs(attributeWrapper.load(), obj) + + def test_failsWithEmptyFQPN(self): + """ + L{wrapFQPN} raises L{InvalidFQPN} when given an empty string. + """ + with self.assertRaises(self.InvalidFQPN): + self.wrapFQPN('') + + def test_failsWithBadDotting(self): + """" + L{wrapFQPN} raises L{InvalidFQPN} when given a badly-dotted + FQPN. (e.g., x..y). + """ + for bad in ('.fails', 'fails.', 'this..fails'): + with self.assertRaises(self.InvalidFQPN): + self.wrapFQPN(bad) + + def test_singleModule(self): + """ + L{wrapFQPN} returns a L{twisted.python.modules.PythonModule} + referring to the single module a dotless FQPN describes. + """ + import os + + moduleWrapper = self.wrapFQPN('os') + + self.assertIsInstance(moduleWrapper, self.PythonModule) + self.assertIs(moduleWrapper.load(), os) + + def test_failsWithMissingSingleModuleOrPackage(self): + """ + L{wrapFQPN} raises L{NoModule} when given a dotless FQPN that does + not refer to a module or package. + """ + with self.assertRaises(self.NoModule): + self.wrapFQPN("this is not an acceptable name!") + + def test_singlePackage(self): + """ + L{wrapFQPN} returns a L{twisted.python.modules.PythonModule} + referring to the single package a dotless FQPN describes. + """ + import xml + self.assertModuleWrapperRefersTo(self.wrapFQPN('xml'), xml) + + def test_multiplePackages(self): + """ + L{wrapFQPN} returns a L{twisted.python.modules.PythonModule} + referring to the deepest package described by dotted FQPN. + """ + import xml.etree + self.assertModuleWrapperRefersTo(self.wrapFQPN('xml.etree'), xml.etree) + + def test_multiplePackagesFinalModule(self): + """ + L{wrapFQPN} returns a L{twisted.python.modules.PythonModule} + referring to the deepest module described by dotted FQPN. + """ + import xml.etree.ElementTree + self.assertModuleWrapperRefersTo( + self.wrapFQPN('xml.etree.ElementTree'), xml.etree.ElementTree) + + def test_singleModuleObject(self): + """ + L{wrapFQPN} returns a L{twisted.python.modules.PythonAttribute} + referring to the deepest object an FQPN names, traversing one module. + """ + import os + self.assertAttributeWrapperRefersTo( + self.wrapFQPN('os.path'), 'os.path', os.path) + + def test_multiplePackagesObject(self): + """ + L{wrapFQPN} returns a L{twisted.python.modules.PythonAttribute} + referring to the deepest object described by an FQPN, + descending through several packages. + """ + import xml.etree.ElementTree + import automat + + for fqpn, obj in [('xml.etree.ElementTree.fromstring', + xml.etree.ElementTree.fromstring), + ('automat.MethodicalMachine.__doc__', + automat.MethodicalMachine.__doc__)]: + self.assertAttributeWrapperRefersTo( + self.wrapFQPN(fqpn), fqpn, obj) + + def test_failsWithMultiplePackagesMissingModuleOrPackage(self): + """ + L{wrapFQPN} raises L{NoObject} when given an FQPN that contains a + missing attribute, module, or package. + """ + for bad in ('xml.etree.nope!', + 'xml.etree.nope!.but.the.rest.is.believable'): + with self.assertRaises(self.NoObject): + self.wrapFQPN(bad) + + +@skipIf(not isTwistedInstalled(), "Twisted is not installed.") +class FindMachinesIntegrationTests(_WritesPythonModules): + """ + Integration tests to check that L{findMachines} yields all + machines discoverable at or below an FQPN. + """ + + SOURCE = """ + from automat import MethodicalMachine + + class PythonClass(object): + _machine = MethodicalMachine() + ignored = "i am ignored" + + rootLevel = MethodicalMachine() + + ignored = "i am ignored" + """ + + def setUp(self): + super(FindMachinesIntegrationTests, self).setUp() + from .._discover import findMachines + + self.findMachines = findMachines + + packageDir = self.FilePath(self.pathDir).child("test_package") + packageDir.makedirs() + self.pythonPath = self.PythonPath([self.pathDir]) + self.writeSourceInto(self.SOURCE, packageDir.path, '__init__.py') + + subPackageDir = packageDir.child('subpackage') + subPackageDir.makedirs() + subPackageDir.child('__init__.py').touch() + + self.makeModule(self.SOURCE, subPackageDir.path, 'module.py') + + self.packageDict = self.loadModuleAsDict( + self.pythonPath['test_package']) + self.moduleDict = self.loadModuleAsDict( + self.pythonPath['test_package']['subpackage']['module']) + + def test_discoverAll(self): + """ + Given a top-level package FQPN, L{findMachines} discovers all + L{MethodicalMachine} instances in and below it. + """ + machines = sorted(self.findMachines('test_package'), + key=operator.itemgetter(0)) + + tpRootLevel = self.packageDict['test_package.rootLevel'].load() + tpPythonClass = self.packageDict['test_package.PythonClass'].load() + + mRLAttr = self.moduleDict['test_package.subpackage.module.rootLevel'] + mRootLevel = mRLAttr.load() + mPCAttr = self.moduleDict['test_package.subpackage.module.PythonClass'] + mPythonClass = mPCAttr.load() + + expectedMachines = sorted( + [('test_package.rootLevel', tpRootLevel), + ('test_package.PythonClass._machine', tpPythonClass._machine), + ('test_package.subpackage.module.rootLevel', mRootLevel), + ('test_package.subpackage.module.PythonClass._machine', + mPythonClass._machine)], + key=operator.itemgetter(0)) + + self.assertEqual(expectedMachines, machines) diff --git a/WebCrawler/venv/Lib/site-packages/automat/_test/test_methodical.py b/WebCrawler/venv/Lib/site-packages/automat/_test/test_methodical.py new file mode 100644 index 0000000..7e3d7ca --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/automat/_test/test_methodical.py @@ -0,0 +1,587 @@ + +""" +Tests for the public interface of Automat. +""" + +from functools import reduce +from unittest import TestCase + +from automat._methodical import ArgSpec, _getArgNames, _getArgSpec, _filterArgs +from .. import MethodicalMachine, NoTransition +from .. import _methodical + + +class MethodicalTests(TestCase): + """ + Tests for L{MethodicalMachine}. + """ + + def test_oneTransition(self): + """ + L{MethodicalMachine} provides a way for you to declare a state machine + with inputs, outputs, and states as methods. When you have declared an + input, an output, and a state, calling the input method in that state + will produce the specified output. + """ + + class Machination(object): + machine = MethodicalMachine() + @machine.input() + def anInput(self): + "an input" + + @machine.output() + def anOutput(self): + "an output" + return "an-output-value" + + @machine.output() + def anotherOutput(self): + "another output" + return "another-output-value" + + @machine.state(initial=True) + def anState(self): + "a state" + + @machine.state() + def anotherState(self): + "another state" + + anState.upon(anInput, enter=anotherState, outputs=[anOutput]) + anotherState.upon(anInput, enter=anotherState, + outputs=[anotherOutput]) + + m = Machination() + self.assertEqual(m.anInput(), ["an-output-value"]) + self.assertEqual(m.anInput(), ["another-output-value"]) + + + def test_machineItselfIsPrivate(self): + """ + L{MethodicalMachine} is an implementation detail. If you attempt to + access it on an instance of your class, you will get an exception. + However, since tools may need to access it for the purposes of, for + example, visualization, you may access it on the class itself. + """ + expectedMachine = MethodicalMachine() + class Machination(object): + machine = expectedMachine + machination = Machination() + with self.assertRaises(AttributeError) as cm: + machination.machine + self.assertIn("MethodicalMachine is an implementation detail", + str(cm.exception)) + self.assertIs(Machination.machine, expectedMachine) + + + def test_outputsArePrivate(self): + """ + One of the benefits of using a state machine is that your output method + implementations don't need to take invalid state transitions into + account - the methods simply won't be called. This property would be + broken if client code called output methods directly, so output methods + are not directly visible under their names. + """ + class Machination(object): + machine = MethodicalMachine() + counter = 0 + @machine.input() + def anInput(self): + "an input" + @machine.output() + def anOutput(self): + self.counter += 1 + @machine.state(initial=True) + def state(self): + "a machine state" + state.upon(anInput, enter=state, outputs=[anOutput]) + mach1 = Machination() + mach1.anInput() + self.assertEqual(mach1.counter, 1) + mach2 = Machination() + with self.assertRaises(AttributeError) as cm: + mach2.anOutput + self.assertEqual(mach2.counter, 0) + + self.assertIn( + "Machination.anOutput is a state-machine output method; to " + "produce this output, call an input method instead.", + str(cm.exception) + ) + + + def test_multipleMachines(self): + """ + Two machines may co-exist happily on the same instance; they don't + interfere with each other. + """ + class MultiMach(object): + a = MethodicalMachine() + b = MethodicalMachine() + + @a.input() + def inputA(self): + "input A" + @b.input() + def inputB(self): + "input B" + @a.state(initial=True) + def initialA(self): + "initial A" + @b.state(initial=True) + def initialB(self): + "initial B" + @a.output() + def outputA(self): + return "A" + @b.output() + def outputB(self): + return "B" + initialA.upon(inputA, initialA, [outputA]) + initialB.upon(inputB, initialB, [outputB]) + + mm = MultiMach() + self.assertEqual(mm.inputA(), ["A"]) + self.assertEqual(mm.inputB(), ["B"]) + + + def test_collectOutputs(self): + """ + Outputs can be combined with the "collector" argument to "upon". + """ + import operator + class Machine(object): + m = MethodicalMachine() + @m.input() + def input(self): + "an input" + @m.output() + def outputA(self): + return "A" + @m.output() + def outputB(self): + return "B" + @m.state(initial=True) + def state(self): + "a state" + state.upon(input, state, [outputA, outputB], + collector=lambda x: reduce(operator.add, x)) + m = Machine() + self.assertEqual(m.input(), "AB") + + + def test_methodName(self): + """ + Input methods preserve their declared names. + """ + class Mech(object): + m = MethodicalMachine() + @m.input() + def declaredInputName(self): + "an input" + @m.state(initial=True) + def aState(self): + "state" + m = Mech() + with self.assertRaises(TypeError) as cm: + m.declaredInputName("too", "many", "arguments") + self.assertIn("declaredInputName", str(cm.exception)) + + + def test_inputWithArguments(self): + """ + If an input takes an argument, it will pass that along to its output. + """ + class Mechanism(object): + m = MethodicalMachine() + @m.input() + def input(self, x, y=1): + "an input" + @m.state(initial=True) + def state(self): + "a state" + @m.output() + def output(self, x, y=1): + self._x = x + return x + y + state.upon(input, state, [output]) + + m = Mechanism() + self.assertEqual(m.input(3), [4]) + self.assertEqual(m._x, 3) + + + def test_outputWithSubsetOfArguments(self): + """ + Inputs pass arguments that output will accept. + """ + class Mechanism(object): + m = MethodicalMachine() + @m.input() + def input(self, x, y=1): + "an input" + @m.state(initial=True) + def state(self): + "a state" + @m.output() + def outputX(self, x): + self._x = x + return x + @m.output() + def outputY(self, y): + self._y = y + return y + @m.output() + def outputNoArgs(self): + return None + state.upon(input, state, [outputX, outputY, outputNoArgs]) + + m = Mechanism() + + # Pass x as positional argument. + self.assertEqual(m.input(3), [3, 1, None]) + self.assertEqual(m._x, 3) + self.assertEqual(m._y, 1) + + # Pass x as key word argument. + self.assertEqual(m.input(x=4), [4, 1, None]) + self.assertEqual(m._x, 4) + self.assertEqual(m._y, 1) + + # Pass y as positional argument. + self.assertEqual(m.input(6, 3), [6, 3, None]) + self.assertEqual(m._x, 6) + self.assertEqual(m._y, 3) + + # Pass y as key word argument. + self.assertEqual(m.input(5, y=2), [5, 2, None]) + self.assertEqual(m._x, 5) + self.assertEqual(m._y, 2) + + + def test_inputFunctionsMustBeEmpty(self): + """ + The wrapped input function must have an empty body. + """ + # input functions are executed to assert that the signature matches, + # but their body must be empty + + _methodical._empty() # chase coverage + _methodical._docstring() + + class Mechanism(object): + m = MethodicalMachine() + with self.assertRaises(ValueError) as cm: + @m.input() + def input(self): + "an input" + list() # pragma: no cover + self.assertEqual(str(cm.exception), "function body must be empty") + + # all three of these cases should be valid. Functions/methods with + # docstrings produce slightly different bytecode than ones without. + + class MechanismWithDocstring(object): + m = MethodicalMachine() + @m.input() + def input(self): + "an input" + @m.state(initial=True) + def start(self): + "starting state" + start.upon(input, enter=start, outputs=[]) + MechanismWithDocstring().input() + + class MechanismWithPass(object): + m = MethodicalMachine() + @m.input() + def input(self): + pass + @m.state(initial=True) + def start(self): + "starting state" + start.upon(input, enter=start, outputs=[]) + MechanismWithPass().input() + + class MechanismWithDocstringAndPass(object): + m = MethodicalMachine() + @m.input() + def input(self): + "an input" + pass + @m.state(initial=True) + def start(self): + "starting state" + start.upon(input, enter=start, outputs=[]) + MechanismWithDocstringAndPass().input() + + class MechanismReturnsNone(object): + m = MethodicalMachine() + @m.input() + def input(self): + return None + @m.state(initial=True) + def start(self): + "starting state" + start.upon(input, enter=start, outputs=[]) + MechanismReturnsNone().input() + + class MechanismWithDocstringAndReturnsNone(object): + m = MethodicalMachine() + @m.input() + def input(self): + "an input" + return None + @m.state(initial=True) + def start(self): + "starting state" + start.upon(input, enter=start, outputs=[]) + MechanismWithDocstringAndReturnsNone().input() + + + def test_inputOutputMismatch(self): + """ + All the argument lists of the outputs for a given input must match; if + one does not the call to C{upon} will raise a C{TypeError}. + """ + class Mechanism(object): + m = MethodicalMachine() + @m.input() + def nameOfInput(self, a): + "an input" + @m.output() + def outputThatMatches(self, a): + "an output that matches" + @m.output() + def outputThatDoesntMatch(self, b): + "an output that doesn't match" + @m.state() + def state(self): + "a state" + with self.assertRaises(TypeError) as cm: + state.upon(nameOfInput, state, [outputThatMatches, + outputThatDoesntMatch]) + self.assertIn("nameOfInput", str(cm.exception)) + self.assertIn("outputThatDoesntMatch", str(cm.exception)) + + + def test_getArgNames(self): + """ + Type annotations should be included in the set of + """ + spec = ArgSpec( + args=('a', 'b'), + varargs=None, + varkw=None, + defaults=None, + kwonlyargs=(), + kwonlydefaults=None, + annotations=(('a', int), ('b', str)), + ) + self.assertEqual( + _getArgNames(spec), + {'a', 'b', ('a', int), ('b', str)}, + ) + + + def test_filterArgs(self): + """ + filterArgs() should not filter the `args` parameter + if outputSpec accepts `*args`. + """ + inputSpec = _getArgSpec(lambda *args, **kwargs: None) + outputSpec = _getArgSpec(lambda *args, **kwargs: None) + argsIn = () + argsOut, _ = _filterArgs(argsIn, {}, inputSpec, outputSpec) + self.assertIs(argsIn, argsOut) + + + def test_multipleInitialStatesFailure(self): + """ + A L{MethodicalMachine} can only have one initial state. + """ + + class WillFail(object): + m = MethodicalMachine() + + @m.state(initial=True) + def firstInitialState(self): + "The first initial state -- this is OK." + + with self.assertRaises(ValueError): + @m.state(initial=True) + def secondInitialState(self): + "The second initial state -- results in a ValueError." + + + def test_multipleTransitionsFailure(self): + """ + A L{MethodicalMachine} can only have one transition per start/event + pair. + """ + + class WillFail(object): + m = MethodicalMachine() + + @m.state(initial=True) + def start(self): + "We start here." + @m.state() + def end(self): + "Rainbows end." + + @m.input() + def event(self): + "An event." + start.upon(event, enter=end, outputs=[]) + with self.assertRaises(ValueError): + start.upon(event, enter=end, outputs=[]) + + + def test_badTransitionForCurrentState(self): + """ + Calling any input method that lacks a transition for the machine's + current state raises an informative L{NoTransition}. + """ + + class OnlyOnePath(object): + m = MethodicalMachine() + @m.state(initial=True) + def start(self): + "Start state." + @m.state() + def end(self): + "End state." + @m.input() + def advance(self): + "Move from start to end." + @m.input() + def deadEnd(self): + "A transition from nowhere to nowhere." + start.upon(advance, end, []) + + machine = OnlyOnePath() + with self.assertRaises(NoTransition) as cm: + machine.deadEnd() + self.assertIn("deadEnd", str(cm.exception)) + self.assertIn("start", str(cm.exception)) + machine.advance() + with self.assertRaises(NoTransition) as cm: + machine.deadEnd() + self.assertIn("deadEnd", str(cm.exception)) + self.assertIn("end", str(cm.exception)) + + + def test_saveState(self): + """ + L{MethodicalMachine.serializer} is a decorator that modifies its + decoratee's signature to take a "state" object as its first argument, + which is the "serialized" argument to the L{MethodicalMachine.state} + decorator. + """ + + class Mechanism(object): + m = MethodicalMachine() + def __init__(self): + self.value = 1 + @m.state(serialized="first-state", initial=True) + def first(self): + "First state." + @m.state(serialized="second-state") + def second(self): + "Second state." + @m.serializer() + def save(self, state): + return { + 'machine-state': state, + 'some-value': self.value, + } + + self.assertEqual( + Mechanism().save(), + { + "machine-state": "first-state", + "some-value": 1, + } + ) + + + def test_restoreState(self): + """ + L{MethodicalMachine.unserializer} decorates a function that becomes a + machine-state unserializer; its return value is mapped to the + C{serialized} parameter to C{state}, and the L{MethodicalMachine} + associated with that instance's state is updated to that state. + """ + + class Mechanism(object): + m = MethodicalMachine() + def __init__(self): + self.value = 1 + self.ranOutput = False + @m.state(serialized="first-state", initial=True) + def first(self): + "First state." + @m.state(serialized="second-state") + def second(self): + "Second state." + @m.input() + def input(self): + "an input" + @m.output() + def output(self): + self.value = 2 + self.ranOutput = True + return 1 + @m.output() + def output2(self): + return 2 + first.upon(input, second, [output], + collector=lambda x: list(x)[0]) + second.upon(input, second, [output2], + collector=lambda x: list(x)[0]) + @m.serializer() + def save(self, state): + return { + 'machine-state': state, + 'some-value': self.value, + } + + @m.unserializer() + def _restore(self, blob): + self.value = blob['some-value'] + return blob['machine-state'] + + @classmethod + def fromBlob(cls, blob): + self = cls() + self._restore(blob) + return self + + m1 = Mechanism() + m1.input() + blob = m1.save() + m2 = Mechanism.fromBlob(blob) + self.assertEqual(m2.ranOutput, False) + self.assertEqual(m2.input(), 2) + self.assertEqual( + m2.save(), + { + 'machine-state': 'second-state', + 'some-value': 2, + } + ) + + + +# FIXME: error for wrong types on any call to _oneTransition +# FIXME: better public API for .upon; maybe a context manager? +# FIXME: when transitions are defined, validate that we can always get to +# terminal? do we care about this? +# FIXME: implementation (and use-case/example) for passing args from in to out + +# FIXME: possibly these need some kind of support from core +# FIXME: wildcard state (in all states, when input X, emit Y and go to Z) +# FIXME: wildcard input (in state X, when any input, emit Y and go to Z) +# FIXME: combined wildcards (in any state for any input, emit Y go to Z) diff --git a/WebCrawler/venv/Lib/site-packages/automat/_test/test_trace.py b/WebCrawler/venv/Lib/site-packages/automat/_test/test_trace.py new file mode 100644 index 0000000..6d7433b --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/automat/_test/test_trace.py @@ -0,0 +1,98 @@ +from unittest import TestCase +from .._methodical import MethodicalMachine + +class SampleObject(object): + mm = MethodicalMachine() + + @mm.state(initial=True) + def begin(self): + "initial state" + @mm.state() + def middle(self): + "middle state" + @mm.state() + def end(self): + "end state" + + @mm.input() + def go1(self): + "sample input" + @mm.input() + def go2(self): + "sample input" + @mm.input() + def back(self): + "sample input" + + @mm.output() + def out(self): + "sample output" + + setTrace = mm._setTrace + + begin.upon(go1, middle, [out]) + middle.upon(go2, end, [out]) + end.upon(back, middle, []) + middle.upon(back, begin, []) + +class TraceTests(TestCase): + def test_only_inputs(self): + traces = [] + def tracer(old_state, input, new_state): + traces.append((old_state, input, new_state)) + return None # "I only care about inputs, not outputs" + s = SampleObject() + s.setTrace(tracer) + + s.go1() + self.assertEqual(traces, [("begin", "go1", "middle"), + ]) + + s.go2() + self.assertEqual(traces, [("begin", "go1", "middle"), + ("middle", "go2", "end"), + ]) + s.setTrace(None) + s.back() + self.assertEqual(traces, [("begin", "go1", "middle"), + ("middle", "go2", "end"), + ]) + s.go2() + self.assertEqual(traces, [("begin", "go1", "middle"), + ("middle", "go2", "end"), + ]) + + def test_inputs_and_outputs(self): + traces = [] + def tracer(old_state, input, new_state): + traces.append((old_state, input, new_state, None)) + def trace_outputs(output): + traces.append((old_state, input, new_state, output)) + return trace_outputs # "I care about outputs too" + s = SampleObject() + s.setTrace(tracer) + + s.go1() + self.assertEqual(traces, [("begin", "go1", "middle", None), + ("begin", "go1", "middle", "out"), + ]) + + s.go2() + self.assertEqual(traces, [("begin", "go1", "middle", None), + ("begin", "go1", "middle", "out"), + ("middle", "go2", "end", None), + ("middle", "go2", "end", "out"), + ]) + s.setTrace(None) + s.back() + self.assertEqual(traces, [("begin", "go1", "middle", None), + ("begin", "go1", "middle", "out"), + ("middle", "go2", "end", None), + ("middle", "go2", "end", "out"), + ]) + s.go2() + self.assertEqual(traces, [("begin", "go1", "middle", None), + ("begin", "go1", "middle", "out"), + ("middle", "go2", "end", None), + ("middle", "go2", "end", "out"), + ]) diff --git a/WebCrawler/venv/Lib/site-packages/automat/_test/test_visualize.py b/WebCrawler/venv/Lib/site-packages/automat/_test/test_visualize.py new file mode 100644 index 0000000..987eb3c --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/automat/_test/test_visualize.py @@ -0,0 +1,430 @@ +from __future__ import print_function +import functools + +import os +import subprocess +from unittest import TestCase, skipIf + +import attr + +from .._methodical import MethodicalMachine + +from .test_discover import isTwistedInstalled + + +def isGraphvizModuleInstalled(): + """ + Is the graphviz Python module installed? + """ + try: + __import__("graphviz") + except ImportError: + return False + else: + return True + + +def isGraphvizInstalled(): + """ + Are the graphviz tools installed? + """ + r, w = os.pipe() + os.close(w) + try: + return not subprocess.call("dot", stdin=r, shell=True) + finally: + os.close(r) + + + +def sampleMachine(): + """ + Create a sample L{MethodicalMachine} with some sample states. + """ + mm = MethodicalMachine() + class SampleObject(object): + @mm.state(initial=True) + def begin(self): + "initial state" + @mm.state() + def end(self): + "end state" + @mm.input() + def go(self): + "sample input" + @mm.output() + def out(self): + "sample output" + begin.upon(go, end, [out]) + so = SampleObject() + so.go() + return mm + + +@skipIf(not isGraphvizModuleInstalled(), "Graphviz module is not installed.") +class ElementMakerTests(TestCase): + """ + L{elementMaker} generates HTML representing the specified element. + """ + + def setUp(self): + from .._visualize import elementMaker + self.elementMaker = elementMaker + + def test_sortsAttrs(self): + """ + L{elementMaker} orders HTML attributes lexicographically. + """ + expected = r'
' + self.assertEqual(expected, + self.elementMaker("div", + b='2', + a='1', + c='3')) + + def test_quotesAttrs(self): + """ + L{elementMaker} quotes HTML attributes according to DOT's quoting rule. + + See U{http://www.graphviz.org/doc/info/lang.html}, footnote 1. + """ + expected = r'
' + self.assertEqual(expected, + self.elementMaker("div", + b='a " quote', + a=1, + c="a string")) + + def test_noAttrs(self): + """ + L{elementMaker} should render an element with no attributes. + """ + expected = r'
' + self.assertEqual(expected, self.elementMaker("div")) + + +@attr.s +class HTMLElement(object): + """Holds an HTML element, as created by elementMaker.""" + name = attr.ib() + children = attr.ib() + attributes = attr.ib() + + +def findElements(element, predicate): + """ + Recursively collect all elements in an L{HTMLElement} tree that + match the optional predicate. + """ + if predicate(element): + return [element] + elif isLeaf(element): + return [] + + return [result + for child in element.children + for result in findElements(child, predicate)] + + +def isLeaf(element): + """ + This HTML element is actually leaf node. + """ + return not isinstance(element, HTMLElement) + + +@skipIf(not isGraphvizModuleInstalled(), "Graphviz module is not installed.") +class TableMakerTests(TestCase): + """ + Tests that ensure L{tableMaker} generates HTML tables usable as + labels in DOT graphs. + + For more information, read the "HTML-Like Labels" section of + U{http://www.graphviz.org/doc/info/shapes.html}. + """ + + def fakeElementMaker(self, name, *children, **attributes): + return HTMLElement(name=name, children=children, attributes=attributes) + + def setUp(self): + from .._visualize import tableMaker + + self.inputLabel = "input label" + self.port = "the port" + self.tableMaker = functools.partial(tableMaker, + _E=self.fakeElementMaker) + + def test_inputLabelRow(self): + """ + The table returned by L{tableMaker} always contains the input + symbol label in its first row, and that row contains one cell + with a port attribute set to the provided port. + """ + + def hasPort(element): + return (not isLeaf(element) + and element.attributes.get("port") == self.port) + + for outputLabels in ([], ["an output label"]): + table = self.tableMaker(self.inputLabel, outputLabels, + port=self.port) + self.assertGreater(len(table.children), 0) + inputLabelRow = table.children[0] + + portCandidates = findElements(table, hasPort) + + self.assertEqual(len(portCandidates), 1) + self.assertEqual(portCandidates[0].name, "td") + self.assertEqual(findElements(inputLabelRow, isLeaf), + [self.inputLabel]) + + def test_noOutputLabels(self): + """ + L{tableMaker} does not add a colspan attribute to the input + label's cell or a second row if there no output labels. + """ + table = self.tableMaker("input label", (), port=self.port) + self.assertEqual(len(table.children), 1) + (inputLabelRow,) = table.children + self.assertNotIn("colspan", inputLabelRow.attributes) + + def test_withOutputLabels(self): + """ + L{tableMaker} adds a colspan attribute to the input label's cell + equal to the number of output labels and a second row that + contains the output labels. + """ + table = self.tableMaker(self.inputLabel, ("output label 1", + "output label 2"), + port=self.port) + + self.assertEqual(len(table.children), 2) + inputRow, outputRow = table.children + + def hasCorrectColspan(element): + return (not isLeaf(element) + and element.name == "td" + and element.attributes.get('colspan') == "2") + + self.assertEqual(len(findElements(inputRow, hasCorrectColspan)), + 1) + self.assertEqual(findElements(outputRow, isLeaf), ["output label 1", + "output label 2"]) + + +@skipIf(not isGraphvizModuleInstalled(), "Graphviz module is not installed.") +@skipIf(not isGraphvizInstalled(), "Graphviz tools are not installed.") +class IntegrationTests(TestCase): + """ + Tests which make sure Graphviz can understand the output produced by + Automat. + """ + + def test_validGraphviz(self): + """ + L{graphviz} emits valid graphviz data. + """ + p = subprocess.Popen("dot", stdin=subprocess.PIPE, + stdout=subprocess.PIPE) + out, err = p.communicate("".join(sampleMachine().asDigraph()) + .encode("utf-8")) + self.assertEqual(p.returncode, 0) + + +@skipIf(not isGraphvizModuleInstalled(), "Graphviz module is not installed.") +class SpotChecks(TestCase): + """ + Tests to make sure that the output contains salient features of the machine + being generated. + """ + + def test_containsMachineFeatures(self): + """ + The output of L{graphviz} should contain the names of the states, + inputs, outputs in the state machine. + """ + gvout = "".join(sampleMachine().asDigraph()) + self.assertIn("begin", gvout) + self.assertIn("end", gvout) + self.assertIn("go", gvout) + self.assertIn("out", gvout) + + +class RecordsDigraphActions(object): + """ + Records calls made to L{FakeDigraph}. + """ + + def __init__(self): + self.reset() + + def reset(self): + self.renderCalls = [] + self.saveCalls = [] + + +class FakeDigraph(object): + """ + A fake L{graphviz.Digraph}. Instantiate it with a + L{RecordsDigraphActions}. + """ + + def __init__(self, recorder): + self._recorder = recorder + + def render(self, **kwargs): + self._recorder.renderCalls.append(kwargs) + + def save(self, **kwargs): + self._recorder.saveCalls.append(kwargs) + + +class FakeMethodicalMachine(object): + """ + A fake L{MethodicalMachine}. Instantiate it with a L{FakeDigraph} + """ + + def __init__(self, digraph): + self._digraph = digraph + + def asDigraph(self): + return self._digraph + + +@skipIf(not isGraphvizModuleInstalled(), "Graphviz module is not installed.") +@skipIf(not isGraphvizInstalled(), "Graphviz tools are not installed.") +@skipIf(not isTwistedInstalled(), "Twisted is not installed.") +class VisualizeToolTests(TestCase): + + def setUp(self): + self.digraphRecorder = RecordsDigraphActions() + self.fakeDigraph = FakeDigraph(self.digraphRecorder) + + self.fakeProgname = 'tool-test' + self.fakeSysPath = ['ignored'] + self.collectedOutput = [] + self.fakeFQPN = 'fake.fqpn' + + def collectPrints(self, *args): + self.collectedOutput.append(' '.join(args)) + + def fakeFindMachines(self, fqpn): + yield fqpn, FakeMethodicalMachine(self.fakeDigraph) + + def tool(self, + progname=None, + argv=None, + syspath=None, + findMachines=None, + print=None): + from .._visualize import tool + return tool( + _progname=progname or self.fakeProgname, + _argv=argv or [self.fakeFQPN], + _syspath=syspath or self.fakeSysPath, + _findMachines=findMachines or self.fakeFindMachines, + _print=print or self.collectPrints) + + def test_checksCurrentDirectory(self): + """ + L{tool} adds '' to sys.path to ensure + L{automat._discover.findMachines} searches the current + directory. + """ + self.tool(argv=[self.fakeFQPN]) + self.assertEqual(self.fakeSysPath[0], '') + + def test_quietHidesOutput(self): + """ + Passing -q/--quiet hides all output. + """ + self.tool(argv=[self.fakeFQPN, '--quiet']) + self.assertFalse(self.collectedOutput) + self.tool(argv=[self.fakeFQPN, '-q']) + self.assertFalse(self.collectedOutput) + + def test_onlySaveDot(self): + """ + Passing an empty string for --image-directory/-i disables + rendering images. + """ + for arg in ('--image-directory', '-i'): + self.digraphRecorder.reset() + self.collectedOutput = [] + + self.tool(argv=[self.fakeFQPN, arg, '']) + self.assertFalse(any("image" in line + for line in self.collectedOutput)) + + self.assertEqual(len(self.digraphRecorder.saveCalls), 1) + (call,) = self.digraphRecorder.saveCalls + self.assertEqual("{}.dot".format(self.fakeFQPN), + call['filename']) + + self.assertFalse(self.digraphRecorder.renderCalls) + + def test_saveOnlyImage(self): + """ + Passing an empty string for --dot-directory/-d disables saving dot + files. + """ + for arg in ('--dot-directory', '-d'): + self.digraphRecorder.reset() + self.collectedOutput = [] + self.tool(argv=[self.fakeFQPN, arg, '']) + + self.assertFalse(any("dot" in line + for line in self.collectedOutput)) + + self.assertEqual(len(self.digraphRecorder.renderCalls), 1) + (call,) = self.digraphRecorder.renderCalls + self.assertEqual("{}.dot".format(self.fakeFQPN), + call['filename']) + self.assertTrue(call['cleanup']) + + self.assertFalse(self.digraphRecorder.saveCalls) + + def test_saveDotAndImagesInDifferentDirectories(self): + """ + Passing different directories to --image-directory and --dot-directory + writes images and dot files to those directories. + """ + imageDirectory = 'image' + dotDirectory = 'dot' + self.tool(argv=[self.fakeFQPN, + '--image-directory', imageDirectory, + '--dot-directory', dotDirectory]) + + self.assertTrue(any("image" in line + for line in self.collectedOutput)) + self.assertTrue(any("dot" in line + for line in self.collectedOutput)) + + self.assertEqual(len(self.digraphRecorder.renderCalls), 1) + (renderCall,) = self.digraphRecorder.renderCalls + self.assertEqual(renderCall["directory"], imageDirectory) + self.assertTrue(renderCall['cleanup']) + + self.assertEqual(len(self.digraphRecorder.saveCalls), 1) + (saveCall,) = self.digraphRecorder.saveCalls + self.assertEqual(saveCall["directory"], dotDirectory) + + def test_saveDotAndImagesInSameDirectory(self): + """ + Passing the same directory to --image-directory and --dot-directory + writes images and dot files to that one directory. + """ + directory = 'imagesAndDot' + self.tool(argv=[self.fakeFQPN, + '--image-directory', directory, + '--dot-directory', directory]) + + self.assertTrue(any("image and dot" in line + for line in self.collectedOutput)) + + self.assertEqual(len(self.digraphRecorder.renderCalls), 1) + (renderCall,) = self.digraphRecorder.renderCalls + self.assertEqual(renderCall["directory"], directory) + self.assertFalse(renderCall['cleanup']) + + self.assertFalse(len(self.digraphRecorder.saveCalls)) diff --git a/WebCrawler/venv/Lib/site-packages/automat/_visualize.py b/WebCrawler/venv/Lib/site-packages/automat/_visualize.py new file mode 100644 index 0000000..7a9c8c6 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/automat/_visualize.py @@ -0,0 +1,182 @@ +from __future__ import print_function +import argparse +import sys + +import graphviz + +from ._discover import findMachines + + +def _gvquote(s): + return '"{}"'.format(s.replace('"', r'\"')) + + +def _gvhtml(s): + return '<{}>'.format(s) + + +def elementMaker(name, *children, **attrs): + """ + Construct a string from the HTML element description. + """ + formattedAttrs = ' '.join('{}={}'.format(key, _gvquote(str(value))) + for key, value in sorted(attrs.items())) + formattedChildren = ''.join(children) + return u'<{name} {attrs}>{children}'.format( + name=name, + attrs=formattedAttrs, + children=formattedChildren) + + +def tableMaker(inputLabel, outputLabels, port, _E=elementMaker): + """ + Construct an HTML table to label a state transition. + """ + colspan = {} + if outputLabels: + colspan['colspan'] = str(len(outputLabels)) + + inputLabelCell = _E("td", + _E("font", + inputLabel, + face="menlo-italic"), + color="purple", + port=port, + **colspan) + + pointSize = {"point-size": "9"} + outputLabelCells = [_E("td", + _E("font", + outputLabel, + **pointSize), + color="pink") + for outputLabel in outputLabels] + + rows = [_E("tr", inputLabelCell)] + + if outputLabels: + rows.append(_E("tr", *outputLabelCells)) + + return _E("table", *rows) + + +def makeDigraph(automaton, inputAsString=repr, + outputAsString=repr, + stateAsString=repr): + """ + Produce a L{graphviz.Digraph} object from an automaton. + """ + digraph = graphviz.Digraph(graph_attr={'pack': 'true', + 'dpi': '100'}, + node_attr={'fontname': 'Menlo'}, + edge_attr={'fontname': 'Menlo'}) + + for state in automaton.states(): + if state is automaton.initialState: + stateShape = "bold" + fontName = "Menlo-Bold" + else: + stateShape = "" + fontName = "Menlo" + digraph.node(stateAsString(state), + fontame=fontName, + shape="ellipse", + style=stateShape, + color="blue") + for n, eachTransition in enumerate(automaton.allTransitions()): + inState, inputSymbol, outState, outputSymbols = eachTransition + thisTransition = "t{}".format(n) + inputLabel = inputAsString(inputSymbol) + + port = "tableport" + table = tableMaker(inputLabel, [outputAsString(outputSymbol) + for outputSymbol in outputSymbols], + port=port) + + digraph.node(thisTransition, + label=_gvhtml(table), margin="0.2", shape="none") + + digraph.edge(stateAsString(inState), + '{}:{}:w'.format(thisTransition, port), + arrowhead="none") + digraph.edge('{}:{}:e'.format(thisTransition, port), + stateAsString(outState)) + + return digraph + + +def tool(_progname=sys.argv[0], + _argv=sys.argv[1:], + _syspath=sys.path, + _findMachines=findMachines, + _print=print): + """ + Entry point for command line utility. + """ + + DESCRIPTION = """ + Visualize automat.MethodicalMachines as graphviz graphs. + """ + EPILOG = """ + You must have the graphviz tool suite installed. Please visit + http://www.graphviz.org for more information. + """ + if _syspath[0]: + _syspath.insert(0, '') + argumentParser = argparse.ArgumentParser( + prog=_progname, + description=DESCRIPTION, + epilog=EPILOG) + argumentParser.add_argument('fqpn', + help="A Fully Qualified Path name" + " representing where to find machines.") + argumentParser.add_argument('--quiet', '-q', + help="suppress output", + default=False, + action="store_true") + argumentParser.add_argument('--dot-directory', '-d', + help="Where to write out .dot files.", + default=".automat_visualize") + argumentParser.add_argument('--image-directory', '-i', + help="Where to write out image files.", + default=".automat_visualize") + argumentParser.add_argument('--image-type', '-t', + help="The image format.", + choices=graphviz.FORMATS, + default='png') + argumentParser.add_argument('--view', '-v', + help="View rendered graphs with" + " default image viewer", + default=False, + action="store_true") + args = argumentParser.parse_args(_argv) + + explicitlySaveDot = (args.dot_directory + and (not args.image_directory + or args.image_directory != args.dot_directory)) + if args.quiet: + def _print(*args): + pass + + for fqpn, machine in _findMachines(args.fqpn): + _print(fqpn, '...discovered') + + digraph = machine.asDigraph() + + if explicitlySaveDot: + digraph.save(filename="{}.dot".format(fqpn), + directory=args.dot_directory) + _print(fqpn, "...wrote dot into", args.dot_directory) + + if args.image_directory: + deleteDot = not args.dot_directory or explicitlySaveDot + digraph.format = args.image_type + digraph.render(filename="{}.dot".format(fqpn), + directory=args.image_directory, + view=args.view, + cleanup=deleteDot) + if deleteDot: + msg = "...wrote image into" + else: + msg = "...wrote image and dot into" + _print(fqpn, msg, args.image_directory) diff --git a/WebCrawler/venv/Lib/site-packages/cffi-1.14.4.dist-info/INSTALLER b/WebCrawler/venv/Lib/site-packages/cffi-1.14.4.dist-info/INSTALLER new file mode 100644 index 0000000..a1b589e --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/cffi-1.14.4.dist-info/INSTALLER @@ -0,0 +1 @@ +pip diff --git a/WebCrawler/venv/Lib/site-packages/cffi-1.14.4.dist-info/LICENSE b/WebCrawler/venv/Lib/site-packages/cffi-1.14.4.dist-info/LICENSE new file mode 100644 index 0000000..29225ee --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/cffi-1.14.4.dist-info/LICENSE @@ -0,0 +1,26 @@ + +Except when otherwise stated (look for LICENSE files in directories or +information at the beginning of each file) all software and +documentation is licensed as follows: + + The MIT License + + Permission is hereby granted, free of charge, to any person + obtaining a copy of this software and associated documentation + files (the "Software"), to deal in the Software without + restriction, including without limitation the rights to use, + copy, modify, merge, publish, distribute, sublicense, and/or + sell copies of the Software, and to permit persons to whom the + Software is furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included + in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + DEALINGS IN THE SOFTWARE. + diff --git a/WebCrawler/venv/Lib/site-packages/cffi-1.14.4.dist-info/METADATA b/WebCrawler/venv/Lib/site-packages/cffi-1.14.4.dist-info/METADATA new file mode 100644 index 0000000..1bb2984 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/cffi-1.14.4.dist-info/METADATA @@ -0,0 +1,37 @@ +Metadata-Version: 2.1 +Name: cffi +Version: 1.14.4 +Summary: Foreign Function Interface for Python calling C code. +Home-page: http://cffi.readthedocs.org +Author: Armin Rigo, Maciej Fijalkowski +Author-email: python-cffi@googlegroups.com +License: MIT +Platform: UNKNOWN +Classifier: Programming Language :: Python +Classifier: Programming Language :: Python :: 2 +Classifier: Programming Language :: Python :: 2.6 +Classifier: Programming Language :: Python :: 2.7 +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3.2 +Classifier: Programming Language :: Python :: 3.3 +Classifier: Programming Language :: Python :: 3.4 +Classifier: Programming Language :: Python :: 3.5 +Classifier: Programming Language :: Python :: 3.6 +Classifier: Programming Language :: Python :: Implementation :: CPython +Classifier: Programming Language :: Python :: Implementation :: PyPy +Classifier: License :: OSI Approved :: MIT License +Requires-Dist: pycparser + + +CFFI +==== + +Foreign Function Interface for Python calling C code. +Please see the `Documentation `_. + +Contact +------- + +`Mailing list `_ + + diff --git a/WebCrawler/venv/Lib/site-packages/cffi-1.14.4.dist-info/RECORD b/WebCrawler/venv/Lib/site-packages/cffi-1.14.4.dist-info/RECORD new file mode 100644 index 0000000..01361d5 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/cffi-1.14.4.dist-info/RECORD @@ -0,0 +1,44 @@ +_cffi_backend.cp39-win_amd64.pyd,sha256=PTrrIv2XqL0v7lNBLOQ0ZsdvIqH9kYt2mrali_hZ1aI,182784 +cffi-1.14.4.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4 +cffi-1.14.4.dist-info/LICENSE,sha256=esEZUOct9bRcUXFqeyLnuzSzJNZ_Bl4pOBUt1HLEgV8,1320 +cffi-1.14.4.dist-info/METADATA,sha256=7KLrKixzeTBGgSzYhkjTJqjRajLP1EkcN_xG_bVShL0,1191 +cffi-1.14.4.dist-info/RECORD,, +cffi-1.14.4.dist-info/WHEEL,sha256=2Kg4PzfJLrLEnxRV1e1jZf0TVEjxVcXZXjp8WtjE4tI,105 +cffi-1.14.4.dist-info/entry_points.txt,sha256=Q9f5C9IpjYxo0d2PK9eUcnkgxHc9pHWwjEMaANPKNCI,76 +cffi-1.14.4.dist-info/top_level.txt,sha256=rE7WR3rZfNKxWI9-jn6hsHCAl7MDkB-FmuQbxWjFehQ,19 +cffi/__init__.py,sha256=cZPh5zl2j5Zp7h2-a8VlqWYWQ1AJ3POlA0aBvDh9TL0,527 +cffi/__pycache__/__init__.cpython-39.pyc,, +cffi/__pycache__/api.cpython-39.pyc,, +cffi/__pycache__/backend_ctypes.cpython-39.pyc,, +cffi/__pycache__/cffi_opcode.cpython-39.pyc,, +cffi/__pycache__/commontypes.cpython-39.pyc,, +cffi/__pycache__/cparser.cpython-39.pyc,, +cffi/__pycache__/error.cpython-39.pyc,, +cffi/__pycache__/ffiplatform.cpython-39.pyc,, +cffi/__pycache__/lock.cpython-39.pyc,, +cffi/__pycache__/model.cpython-39.pyc,, +cffi/__pycache__/pkgconfig.cpython-39.pyc,, +cffi/__pycache__/recompiler.cpython-39.pyc,, +cffi/__pycache__/setuptools_ext.cpython-39.pyc,, +cffi/__pycache__/vengine_cpy.cpython-39.pyc,, +cffi/__pycache__/vengine_gen.cpython-39.pyc,, +cffi/__pycache__/verifier.cpython-39.pyc,, +cffi/_cffi_errors.h,sha256=INd0GxZQna8TTRYNOOr9_iFy0FZa84I_KH1qlmPgulQ,4003 +cffi/_cffi_include.h,sha256=H7cgdZR-POwmUFrIup4jOGzmje8YoQHhN99gVFg7w08,15185 +cffi/_embedding.h,sha256=jg_yr7a0FTaws4Q91B-6G134zgSe8LhoBt3-eN9dZj8,18108 +cffi/api.py,sha256=Xs_dAN5x1ehfnn_F9ZTdA3Ce0bmPrqeIOkO4Ya1tfbQ,43029 +cffi/backend_ctypes.py,sha256=BHN3q2giL2_Y8wMDST2CIcc_qoMrs65qV9Ob5JvxBZ4,43575 +cffi/cffi_opcode.py,sha256=57P2NHLZkuTWueZybu5iosWljb6ocQmUXzGrCplrnyE,5911 +cffi/commontypes.py,sha256=mEZD4g0qtadnv6O6CEXvMQaJ1K6SRbG5S1h4YvVZHOU,2769 +cffi/cparser.py,sha256=CwVk2V3ATYlCoywG6zN35w6UQ7zj2EWX68KjoJp2Mzk,45237 +cffi/error.py,sha256=Bka7fSV22aIglTQDPIDfpnxTc1aWZLMQdQOJY-h_PUA,908 +cffi/ffiplatform.py,sha256=qioydJeC63dEvrQ3ht5_BPmSs7wzzzuWnZAJtfhic7I,4173 +cffi/lock.py,sha256=vnbsel7392Ib8gGBifIfAfc7MHteSwd3nP725pvc25Q,777 +cffi/model.py,sha256=HRD0WEYHF2Vr6RjS-4wyncElrZxU2256zY0fbMkSKec,22385 +cffi/parse_c_type.h,sha256=fKYNqWNX5f9kZNNhbXcRLTOlpRGRhh8eCLyHmTXIZnQ,6157 +cffi/pkgconfig.py,sha256=9zDcDf0XKIJaxFHLg7e-W8-Xb8Yq5hdhqH7kLg-ugRo,4495 +cffi/recompiler.py,sha256=LmEalHqs90dgp5od-BiZizsu2M2WJV7S6ctNSxj3FsA,66149 +cffi/setuptools_ext.py,sha256=8y14TOlRAkgdczmwtPOahyFXJHNyIqhLjUHMYQmjOHs,9150 +cffi/vengine_cpy.py,sha256=ukugKCIsURxJzHxlxS265tGjQfPTFDbThwsqBrwKh-A,44396 +cffi/vengine_gen.py,sha256=mykUhLFJIcV6AyQ5cMJ3n_7dbqw0a9WEjXW0E-WfgiI,27359 +cffi/verifier.py,sha256=La8rdbEkvdvbqAHDzTk5lsNUvdkqB_GcFnO7wXI6Mgk,11513 diff --git a/WebCrawler/venv/Lib/site-packages/cffi-1.14.4.dist-info/WHEEL b/WebCrawler/venv/Lib/site-packages/cffi-1.14.4.dist-info/WHEEL new file mode 100644 index 0000000..358414f --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/cffi-1.14.4.dist-info/WHEEL @@ -0,0 +1,5 @@ +Wheel-Version: 1.0 +Generator: bdist_wheel (0.35.1) +Root-Is-Purelib: false +Tag: cp39-cp39-win_amd64 + diff --git a/WebCrawler/venv/Lib/site-packages/cffi-1.14.4.dist-info/entry_points.txt b/WebCrawler/venv/Lib/site-packages/cffi-1.14.4.dist-info/entry_points.txt new file mode 100644 index 0000000..eee7e0f --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/cffi-1.14.4.dist-info/entry_points.txt @@ -0,0 +1,3 @@ +[distutils.setup_keywords] +cffi_modules = cffi.setuptools_ext:cffi_modules + diff --git a/WebCrawler/venv/Lib/site-packages/cffi-1.14.4.dist-info/top_level.txt b/WebCrawler/venv/Lib/site-packages/cffi-1.14.4.dist-info/top_level.txt new file mode 100644 index 0000000..f645779 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/cffi-1.14.4.dist-info/top_level.txt @@ -0,0 +1,2 @@ +_cffi_backend +cffi diff --git a/WebCrawler/venv/Lib/site-packages/cffi/__init__.py b/WebCrawler/venv/Lib/site-packages/cffi/__init__.py new file mode 100644 index 0000000..644dea7 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/cffi/__init__.py @@ -0,0 +1,14 @@ +__all__ = ['FFI', 'VerificationError', 'VerificationMissing', 'CDefError', + 'FFIError'] + +from .api import FFI +from .error import CDefError, FFIError, VerificationError, VerificationMissing +from .error import PkgConfigError + +__version__ = "1.14.4" +__version_info__ = (1, 14, 4) + +# The verifier module file names are based on the CRC32 of a string that +# contains the following version number. It may be older than __version__ +# if nothing is clearly incompatible. +__version_verifier_modules__ = "0.8.6" diff --git a/WebCrawler/venv/Lib/site-packages/cffi/__pycache__/__init__.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/cffi/__pycache__/__init__.cpython-39.pyc new file mode 100644 index 0000000..238b3e1 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/cffi/__pycache__/__init__.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/cffi/__pycache__/api.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/cffi/__pycache__/api.cpython-39.pyc new file mode 100644 index 0000000..243fde9 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/cffi/__pycache__/api.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/cffi/__pycache__/backend_ctypes.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/cffi/__pycache__/backend_ctypes.cpython-39.pyc new file mode 100644 index 0000000..456d119 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/cffi/__pycache__/backend_ctypes.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/cffi/__pycache__/cffi_opcode.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/cffi/__pycache__/cffi_opcode.cpython-39.pyc new file mode 100644 index 0000000..ee2111d Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/cffi/__pycache__/cffi_opcode.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/cffi/__pycache__/commontypes.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/cffi/__pycache__/commontypes.cpython-39.pyc new file mode 100644 index 0000000..ba21abe Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/cffi/__pycache__/commontypes.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/cffi/__pycache__/cparser.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/cffi/__pycache__/cparser.cpython-39.pyc new file mode 100644 index 0000000..dfaa001 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/cffi/__pycache__/cparser.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/cffi/__pycache__/error.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/cffi/__pycache__/error.cpython-39.pyc new file mode 100644 index 0000000..719a37e Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/cffi/__pycache__/error.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/cffi/__pycache__/ffiplatform.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/cffi/__pycache__/ffiplatform.cpython-39.pyc new file mode 100644 index 0000000..b76cecd Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/cffi/__pycache__/ffiplatform.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/cffi/__pycache__/lock.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/cffi/__pycache__/lock.cpython-39.pyc new file mode 100644 index 0000000..8a22fd4 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/cffi/__pycache__/lock.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/cffi/__pycache__/model.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/cffi/__pycache__/model.cpython-39.pyc new file mode 100644 index 0000000..2e82820 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/cffi/__pycache__/model.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/cffi/__pycache__/pkgconfig.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/cffi/__pycache__/pkgconfig.cpython-39.pyc new file mode 100644 index 0000000..c1846ce Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/cffi/__pycache__/pkgconfig.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/cffi/__pycache__/recompiler.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/cffi/__pycache__/recompiler.cpython-39.pyc new file mode 100644 index 0000000..3266c18 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/cffi/__pycache__/recompiler.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/cffi/__pycache__/setuptools_ext.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/cffi/__pycache__/setuptools_ext.cpython-39.pyc new file mode 100644 index 0000000..e1d62cb Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/cffi/__pycache__/setuptools_ext.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/cffi/__pycache__/vengine_cpy.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/cffi/__pycache__/vengine_cpy.cpython-39.pyc new file mode 100644 index 0000000..1d2548a Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/cffi/__pycache__/vengine_cpy.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/cffi/__pycache__/vengine_gen.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/cffi/__pycache__/vengine_gen.cpython-39.pyc new file mode 100644 index 0000000..10dbaef Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/cffi/__pycache__/vengine_gen.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/cffi/__pycache__/verifier.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/cffi/__pycache__/verifier.cpython-39.pyc new file mode 100644 index 0000000..96f86b6 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/cffi/__pycache__/verifier.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/cffi/_cffi_errors.h b/WebCrawler/venv/Lib/site-packages/cffi/_cffi_errors.h new file mode 100644 index 0000000..83cdad0 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/cffi/_cffi_errors.h @@ -0,0 +1,147 @@ +#ifndef CFFI_MESSAGEBOX +# ifdef _MSC_VER +# define CFFI_MESSAGEBOX 1 +# else +# define CFFI_MESSAGEBOX 0 +# endif +#endif + + +#if CFFI_MESSAGEBOX +/* Windows only: logic to take the Python-CFFI embedding logic + initialization errors and display them in a background thread + with MessageBox. The idea is that if the whole program closes + as a result of this problem, then likely it is already a console + program and you can read the stderr output in the console too. + If it is not a console program, then it will likely show its own + dialog to complain, or generally not abruptly close, and for this + case the background thread should stay alive. +*/ +static void *volatile _cffi_bootstrap_text; + +static PyObject *_cffi_start_error_capture(void) +{ + PyObject *result = NULL; + PyObject *x, *m, *bi; + + if (InterlockedCompareExchangePointer(&_cffi_bootstrap_text, + (void *)1, NULL) != NULL) + return (PyObject *)1; + + m = PyImport_AddModule("_cffi_error_capture"); + if (m == NULL) + goto error; + + result = PyModule_GetDict(m); + if (result == NULL) + goto error; + +#if PY_MAJOR_VERSION >= 3 + bi = PyImport_ImportModule("builtins"); +#else + bi = PyImport_ImportModule("__builtin__"); +#endif + if (bi == NULL) + goto error; + PyDict_SetItemString(result, "__builtins__", bi); + Py_DECREF(bi); + + x = PyRun_String( + "import sys\n" + "class FileLike:\n" + " def write(self, x):\n" + " try:\n" + " of.write(x)\n" + " except: pass\n" + " self.buf += x\n" + "fl = FileLike()\n" + "fl.buf = ''\n" + "of = sys.stderr\n" + "sys.stderr = fl\n" + "def done():\n" + " sys.stderr = of\n" + " return fl.buf\n", /* make sure the returned value stays alive */ + Py_file_input, + result, result); + Py_XDECREF(x); + + error: + if (PyErr_Occurred()) + { + PyErr_WriteUnraisable(Py_None); + PyErr_Clear(); + } + return result; +} + +#pragma comment(lib, "user32.lib") + +static DWORD WINAPI _cffi_bootstrap_dialog(LPVOID ignored) +{ + Sleep(666); /* may be interrupted if the whole process is closing */ +#if PY_MAJOR_VERSION >= 3 + MessageBoxW(NULL, (wchar_t *)_cffi_bootstrap_text, + L"Python-CFFI error", + MB_OK | MB_ICONERROR); +#else + MessageBoxA(NULL, (char *)_cffi_bootstrap_text, + "Python-CFFI error", + MB_OK | MB_ICONERROR); +#endif + _cffi_bootstrap_text = NULL; + return 0; +} + +static void _cffi_stop_error_capture(PyObject *ecap) +{ + PyObject *s; + void *text; + + if (ecap == (PyObject *)1) + return; + + if (ecap == NULL) + goto error; + + s = PyRun_String("done()", Py_eval_input, ecap, ecap); + if (s == NULL) + goto error; + + /* Show a dialog box, but in a background thread, and + never show multiple dialog boxes at once. */ +#if PY_MAJOR_VERSION >= 3 + text = PyUnicode_AsWideCharString(s, NULL); +#else + text = PyString_AsString(s); +#endif + + _cffi_bootstrap_text = text; + + if (text != NULL) + { + HANDLE h; + h = CreateThread(NULL, 0, _cffi_bootstrap_dialog, + NULL, 0, NULL); + if (h != NULL) + CloseHandle(h); + } + /* decref the string, but it should stay alive as 'fl.buf' + in the small module above. It will really be freed only if + we later get another similar error. So it's a leak of at + most one copy of the small module. That's fine for this + situation which is usually a "fatal error" anyway. */ + Py_DECREF(s); + PyErr_Clear(); + return; + + error: + _cffi_bootstrap_text = NULL; + PyErr_Clear(); +} + +#else + +static PyObject *_cffi_start_error_capture(void) { return NULL; } +static void _cffi_stop_error_capture(PyObject *ecap) { } + +#endif diff --git a/WebCrawler/venv/Lib/site-packages/cffi/_cffi_include.h b/WebCrawler/venv/Lib/site-packages/cffi/_cffi_include.h new file mode 100644 index 0000000..e4c0a67 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/cffi/_cffi_include.h @@ -0,0 +1,385 @@ +#define _CFFI_ + +/* We try to define Py_LIMITED_API before including Python.h. + + Mess: we can only define it if Py_DEBUG, Py_TRACE_REFS and + Py_REF_DEBUG are not defined. This is a best-effort approximation: + we can learn about Py_DEBUG from pyconfig.h, but it is unclear if + the same works for the other two macros. Py_DEBUG implies them, + but not the other way around. + + The implementation is messy (issue #350): on Windows, with _MSC_VER, + we have to define Py_LIMITED_API even before including pyconfig.h. + In that case, we guess what pyconfig.h will do to the macros above, + and check our guess after the #include. + + Note that on Windows, with CPython 3.x, you need >= 3.5 and virtualenv + version >= 16.0.0. With older versions of either, you don't get a + copy of PYTHON3.DLL in the virtualenv. We can't check the version of + CPython *before* we even include pyconfig.h. ffi.set_source() puts + a ``#define _CFFI_NO_LIMITED_API'' at the start of this file if it is + running on Windows < 3.5, as an attempt at fixing it, but that's + arguably wrong because it may not be the target version of Python. + Still better than nothing I guess. As another workaround, you can + remove the definition of Py_LIMITED_API here. + + See also 'py_limited_api' in cffi/setuptools_ext.py. +*/ +#if !defined(_CFFI_USE_EMBEDDING) && !defined(Py_LIMITED_API) +# ifdef _MSC_VER +# if !defined(_DEBUG) && !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) && !defined(Py_REF_DEBUG) && !defined(_CFFI_NO_LIMITED_API) +# define Py_LIMITED_API +# endif +# include + /* sanity-check: Py_LIMITED_API will cause crashes if any of these + are also defined. Normally, the Python file PC/pyconfig.h does not + cause any of these to be defined, with the exception that _DEBUG + causes Py_DEBUG. Double-check that. */ +# ifdef Py_LIMITED_API +# if defined(Py_DEBUG) +# error "pyconfig.h unexpectedly defines Py_DEBUG, but Py_LIMITED_API is set" +# endif +# if defined(Py_TRACE_REFS) +# error "pyconfig.h unexpectedly defines Py_TRACE_REFS, but Py_LIMITED_API is set" +# endif +# if defined(Py_REF_DEBUG) +# error "pyconfig.h unexpectedly defines Py_REF_DEBUG, but Py_LIMITED_API is set" +# endif +# endif +# else +# include +# if !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) && !defined(Py_REF_DEBUG) && !defined(_CFFI_NO_LIMITED_API) +# define Py_LIMITED_API +# endif +# endif +#endif + +#include +#ifdef __cplusplus +extern "C" { +#endif +#include +#include "parse_c_type.h" + +/* this block of #ifs should be kept exactly identical between + c/_cffi_backend.c, cffi/vengine_cpy.py, cffi/vengine_gen.py + and cffi/_cffi_include.h */ +#if defined(_MSC_VER) +# include /* for alloca() */ +# if _MSC_VER < 1600 /* MSVC < 2010 */ + typedef __int8 int8_t; + typedef __int16 int16_t; + typedef __int32 int32_t; + typedef __int64 int64_t; + typedef unsigned __int8 uint8_t; + typedef unsigned __int16 uint16_t; + typedef unsigned __int32 uint32_t; + typedef unsigned __int64 uint64_t; + typedef __int8 int_least8_t; + typedef __int16 int_least16_t; + typedef __int32 int_least32_t; + typedef __int64 int_least64_t; + typedef unsigned __int8 uint_least8_t; + typedef unsigned __int16 uint_least16_t; + typedef unsigned __int32 uint_least32_t; + typedef unsigned __int64 uint_least64_t; + typedef __int8 int_fast8_t; + typedef __int16 int_fast16_t; + typedef __int32 int_fast32_t; + typedef __int64 int_fast64_t; + typedef unsigned __int8 uint_fast8_t; + typedef unsigned __int16 uint_fast16_t; + typedef unsigned __int32 uint_fast32_t; + typedef unsigned __int64 uint_fast64_t; + typedef __int64 intmax_t; + typedef unsigned __int64 uintmax_t; +# else +# include +# endif +# if _MSC_VER < 1800 /* MSVC < 2013 */ +# ifndef __cplusplus + typedef unsigned char _Bool; +# endif +# endif +#else +# include +# if (defined (__SVR4) && defined (__sun)) || defined(_AIX) || defined(__hpux) +# include +# endif +#endif + +#ifdef __GNUC__ +# define _CFFI_UNUSED_FN __attribute__((unused)) +#else +# define _CFFI_UNUSED_FN /* nothing */ +#endif + +#ifdef __cplusplus +# ifndef _Bool + typedef bool _Bool; /* semi-hackish: C++ has no _Bool; bool is builtin */ +# endif +#endif + +/********** CPython-specific section **********/ +#ifndef PYPY_VERSION + + +#if PY_MAJOR_VERSION >= 3 +# define PyInt_FromLong PyLong_FromLong +#endif + +#define _cffi_from_c_double PyFloat_FromDouble +#define _cffi_from_c_float PyFloat_FromDouble +#define _cffi_from_c_long PyInt_FromLong +#define _cffi_from_c_ulong PyLong_FromUnsignedLong +#define _cffi_from_c_longlong PyLong_FromLongLong +#define _cffi_from_c_ulonglong PyLong_FromUnsignedLongLong +#define _cffi_from_c__Bool PyBool_FromLong + +#define _cffi_to_c_double PyFloat_AsDouble +#define _cffi_to_c_float PyFloat_AsDouble + +#define _cffi_from_c_int(x, type) \ + (((type)-1) > 0 ? /* unsigned */ \ + (sizeof(type) < sizeof(long) ? \ + PyInt_FromLong((long)x) : \ + sizeof(type) == sizeof(long) ? \ + PyLong_FromUnsignedLong((unsigned long)x) : \ + PyLong_FromUnsignedLongLong((unsigned long long)x)) : \ + (sizeof(type) <= sizeof(long) ? \ + PyInt_FromLong((long)x) : \ + PyLong_FromLongLong((long long)x))) + +#define _cffi_to_c_int(o, type) \ + ((type)( \ + sizeof(type) == 1 ? (((type)-1) > 0 ? (type)_cffi_to_c_u8(o) \ + : (type)_cffi_to_c_i8(o)) : \ + sizeof(type) == 2 ? (((type)-1) > 0 ? (type)_cffi_to_c_u16(o) \ + : (type)_cffi_to_c_i16(o)) : \ + sizeof(type) == 4 ? (((type)-1) > 0 ? (type)_cffi_to_c_u32(o) \ + : (type)_cffi_to_c_i32(o)) : \ + sizeof(type) == 8 ? (((type)-1) > 0 ? (type)_cffi_to_c_u64(o) \ + : (type)_cffi_to_c_i64(o)) : \ + (Py_FatalError("unsupported size for type " #type), (type)0))) + +#define _cffi_to_c_i8 \ + ((int(*)(PyObject *))_cffi_exports[1]) +#define _cffi_to_c_u8 \ + ((int(*)(PyObject *))_cffi_exports[2]) +#define _cffi_to_c_i16 \ + ((int(*)(PyObject *))_cffi_exports[3]) +#define _cffi_to_c_u16 \ + ((int(*)(PyObject *))_cffi_exports[4]) +#define _cffi_to_c_i32 \ + ((int(*)(PyObject *))_cffi_exports[5]) +#define _cffi_to_c_u32 \ + ((unsigned int(*)(PyObject *))_cffi_exports[6]) +#define _cffi_to_c_i64 \ + ((long long(*)(PyObject *))_cffi_exports[7]) +#define _cffi_to_c_u64 \ + ((unsigned long long(*)(PyObject *))_cffi_exports[8]) +#define _cffi_to_c_char \ + ((int(*)(PyObject *))_cffi_exports[9]) +#define _cffi_from_c_pointer \ + ((PyObject *(*)(char *, struct _cffi_ctypedescr *))_cffi_exports[10]) +#define _cffi_to_c_pointer \ + ((char *(*)(PyObject *, struct _cffi_ctypedescr *))_cffi_exports[11]) +#define _cffi_get_struct_layout \ + not used any more +#define _cffi_restore_errno \ + ((void(*)(void))_cffi_exports[13]) +#define _cffi_save_errno \ + ((void(*)(void))_cffi_exports[14]) +#define _cffi_from_c_char \ + ((PyObject *(*)(char))_cffi_exports[15]) +#define _cffi_from_c_deref \ + ((PyObject *(*)(char *, struct _cffi_ctypedescr *))_cffi_exports[16]) +#define _cffi_to_c \ + ((int(*)(char *, struct _cffi_ctypedescr *, PyObject *))_cffi_exports[17]) +#define _cffi_from_c_struct \ + ((PyObject *(*)(char *, struct _cffi_ctypedescr *))_cffi_exports[18]) +#define _cffi_to_c_wchar_t \ + ((_cffi_wchar_t(*)(PyObject *))_cffi_exports[19]) +#define _cffi_from_c_wchar_t \ + ((PyObject *(*)(_cffi_wchar_t))_cffi_exports[20]) +#define _cffi_to_c_long_double \ + ((long double(*)(PyObject *))_cffi_exports[21]) +#define _cffi_to_c__Bool \ + ((_Bool(*)(PyObject *))_cffi_exports[22]) +#define _cffi_prepare_pointer_call_argument \ + ((Py_ssize_t(*)(struct _cffi_ctypedescr *, \ + PyObject *, char **))_cffi_exports[23]) +#define _cffi_convert_array_from_object \ + ((int(*)(char *, struct _cffi_ctypedescr *, PyObject *))_cffi_exports[24]) +#define _CFFI_CPIDX 25 +#define _cffi_call_python \ + ((void(*)(struct _cffi_externpy_s *, char *))_cffi_exports[_CFFI_CPIDX]) +#define _cffi_to_c_wchar3216_t \ + ((int(*)(PyObject *))_cffi_exports[26]) +#define _cffi_from_c_wchar3216_t \ + ((PyObject *(*)(int))_cffi_exports[27]) +#define _CFFI_NUM_EXPORTS 28 + +struct _cffi_ctypedescr; + +static void *_cffi_exports[_CFFI_NUM_EXPORTS]; + +#define _cffi_type(index) ( \ + assert((((uintptr_t)_cffi_types[index]) & 1) == 0), \ + (struct _cffi_ctypedescr *)_cffi_types[index]) + +static PyObject *_cffi_init(const char *module_name, Py_ssize_t version, + const struct _cffi_type_context_s *ctx) +{ + PyObject *module, *o_arg, *new_module; + void *raw[] = { + (void *)module_name, + (void *)version, + (void *)_cffi_exports, + (void *)ctx, + }; + + module = PyImport_ImportModule("_cffi_backend"); + if (module == NULL) + goto failure; + + o_arg = PyLong_FromVoidPtr((void *)raw); + if (o_arg == NULL) + goto failure; + + new_module = PyObject_CallMethod( + module, (char *)"_init_cffi_1_0_external_module", (char *)"O", o_arg); + + Py_DECREF(o_arg); + Py_DECREF(module); + return new_module; + + failure: + Py_XDECREF(module); + return NULL; +} + + +#ifdef HAVE_WCHAR_H +typedef wchar_t _cffi_wchar_t; +#else +typedef uint16_t _cffi_wchar_t; /* same random pick as _cffi_backend.c */ +#endif + +_CFFI_UNUSED_FN static uint16_t _cffi_to_c_char16_t(PyObject *o) +{ + if (sizeof(_cffi_wchar_t) == 2) + return (uint16_t)_cffi_to_c_wchar_t(o); + else + return (uint16_t)_cffi_to_c_wchar3216_t(o); +} + +_CFFI_UNUSED_FN static PyObject *_cffi_from_c_char16_t(uint16_t x) +{ + if (sizeof(_cffi_wchar_t) == 2) + return _cffi_from_c_wchar_t((_cffi_wchar_t)x); + else + return _cffi_from_c_wchar3216_t((int)x); +} + +_CFFI_UNUSED_FN static int _cffi_to_c_char32_t(PyObject *o) +{ + if (sizeof(_cffi_wchar_t) == 4) + return (int)_cffi_to_c_wchar_t(o); + else + return (int)_cffi_to_c_wchar3216_t(o); +} + +_CFFI_UNUSED_FN static PyObject *_cffi_from_c_char32_t(unsigned int x) +{ + if (sizeof(_cffi_wchar_t) == 4) + return _cffi_from_c_wchar_t((_cffi_wchar_t)x); + else + return _cffi_from_c_wchar3216_t((int)x); +} + +union _cffi_union_alignment_u { + unsigned char m_char; + unsigned short m_short; + unsigned int m_int; + unsigned long m_long; + unsigned long long m_longlong; + float m_float; + double m_double; + long double m_longdouble; +}; + +struct _cffi_freeme_s { + struct _cffi_freeme_s *next; + union _cffi_union_alignment_u alignment; +}; + +_CFFI_UNUSED_FN static int +_cffi_convert_array_argument(struct _cffi_ctypedescr *ctptr, PyObject *arg, + char **output_data, Py_ssize_t datasize, + struct _cffi_freeme_s **freeme) +{ + char *p; + if (datasize < 0) + return -1; + + p = *output_data; + if (p == NULL) { + struct _cffi_freeme_s *fp = (struct _cffi_freeme_s *)PyObject_Malloc( + offsetof(struct _cffi_freeme_s, alignment) + (size_t)datasize); + if (fp == NULL) + return -1; + fp->next = *freeme; + *freeme = fp; + p = *output_data = (char *)&fp->alignment; + } + memset((void *)p, 0, (size_t)datasize); + return _cffi_convert_array_from_object(p, ctptr, arg); +} + +_CFFI_UNUSED_FN static void +_cffi_free_array_arguments(struct _cffi_freeme_s *freeme) +{ + do { + void *p = (void *)freeme; + freeme = freeme->next; + PyObject_Free(p); + } while (freeme != NULL); +} + +/********** end CPython-specific section **********/ +#else +_CFFI_UNUSED_FN +static void (*_cffi_call_python_org)(struct _cffi_externpy_s *, char *); +# define _cffi_call_python _cffi_call_python_org +#endif + + +#define _cffi_array_len(array) (sizeof(array) / sizeof((array)[0])) + +#define _cffi_prim_int(size, sign) \ + ((size) == 1 ? ((sign) ? _CFFI_PRIM_INT8 : _CFFI_PRIM_UINT8) : \ + (size) == 2 ? ((sign) ? _CFFI_PRIM_INT16 : _CFFI_PRIM_UINT16) : \ + (size) == 4 ? ((sign) ? _CFFI_PRIM_INT32 : _CFFI_PRIM_UINT32) : \ + (size) == 8 ? ((sign) ? _CFFI_PRIM_INT64 : _CFFI_PRIM_UINT64) : \ + _CFFI__UNKNOWN_PRIM) + +#define _cffi_prim_float(size) \ + ((size) == sizeof(float) ? _CFFI_PRIM_FLOAT : \ + (size) == sizeof(double) ? _CFFI_PRIM_DOUBLE : \ + (size) == sizeof(long double) ? _CFFI__UNKNOWN_LONG_DOUBLE : \ + _CFFI__UNKNOWN_FLOAT_PRIM) + +#define _cffi_check_int(got, got_nonpos, expected) \ + ((got_nonpos) == (expected <= 0) && \ + (got) == (unsigned long long)expected) + +#ifdef MS_WIN32 +# define _cffi_stdcall __stdcall +#else +# define _cffi_stdcall /* nothing */ +#endif + +#ifdef __cplusplus +} +#endif diff --git a/WebCrawler/venv/Lib/site-packages/cffi/_embedding.h b/WebCrawler/venv/Lib/site-packages/cffi/_embedding.h new file mode 100644 index 0000000..cae179a --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/cffi/_embedding.h @@ -0,0 +1,527 @@ + +/***** Support code for embedding *****/ + +#ifdef __cplusplus +extern "C" { +#endif + + +#if defined(_WIN32) +# define CFFI_DLLEXPORT __declspec(dllexport) +#elif defined(__GNUC__) +# define CFFI_DLLEXPORT __attribute__((visibility("default"))) +#else +# define CFFI_DLLEXPORT /* nothing */ +#endif + + +/* There are two global variables of type _cffi_call_python_fnptr: + + * _cffi_call_python, which we declare just below, is the one called + by ``extern "Python"`` implementations. + + * _cffi_call_python_org, which on CPython is actually part of the + _cffi_exports[] array, is the function pointer copied from + _cffi_backend. + + After initialization is complete, both are equal. However, the + first one remains equal to &_cffi_start_and_call_python until the + very end of initialization, when we are (or should be) sure that + concurrent threads also see a completely initialized world, and + only then is it changed. +*/ +#undef _cffi_call_python +typedef void (*_cffi_call_python_fnptr)(struct _cffi_externpy_s *, char *); +static void _cffi_start_and_call_python(struct _cffi_externpy_s *, char *); +static _cffi_call_python_fnptr _cffi_call_python = &_cffi_start_and_call_python; + + +#ifndef _MSC_VER + /* --- Assuming a GCC not infinitely old --- */ +# define cffi_compare_and_swap(l,o,n) __sync_bool_compare_and_swap(l,o,n) +# define cffi_write_barrier() __sync_synchronize() +# if !defined(__amd64__) && !defined(__x86_64__) && \ + !defined(__i386__) && !defined(__i386) +# define cffi_read_barrier() __sync_synchronize() +# else +# define cffi_read_barrier() (void)0 +# endif +#else + /* --- Windows threads version --- */ +# include +# define cffi_compare_and_swap(l,o,n) \ + (InterlockedCompareExchangePointer(l,n,o) == (o)) +# define cffi_write_barrier() InterlockedCompareExchange(&_cffi_dummy,0,0) +# define cffi_read_barrier() (void)0 +static volatile LONG _cffi_dummy; +#endif + +#ifdef WITH_THREAD +# ifndef _MSC_VER +# include + static pthread_mutex_t _cffi_embed_startup_lock; +# else + static CRITICAL_SECTION _cffi_embed_startup_lock; +# endif + static char _cffi_embed_startup_lock_ready = 0; +#endif + +static void _cffi_acquire_reentrant_mutex(void) +{ + static void *volatile lock = NULL; + + while (!cffi_compare_and_swap(&lock, NULL, (void *)1)) { + /* should ideally do a spin loop instruction here, but + hard to do it portably and doesn't really matter I + think: pthread_mutex_init() should be very fast, and + this is only run at start-up anyway. */ + } + +#ifdef WITH_THREAD + if (!_cffi_embed_startup_lock_ready) { +# ifndef _MSC_VER + pthread_mutexattr_t attr; + pthread_mutexattr_init(&attr); + pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE); + pthread_mutex_init(&_cffi_embed_startup_lock, &attr); +# else + InitializeCriticalSection(&_cffi_embed_startup_lock); +# endif + _cffi_embed_startup_lock_ready = 1; + } +#endif + + while (!cffi_compare_and_swap(&lock, (void *)1, NULL)) + ; + +#ifndef _MSC_VER + pthread_mutex_lock(&_cffi_embed_startup_lock); +#else + EnterCriticalSection(&_cffi_embed_startup_lock); +#endif +} + +static void _cffi_release_reentrant_mutex(void) +{ +#ifndef _MSC_VER + pthread_mutex_unlock(&_cffi_embed_startup_lock); +#else + LeaveCriticalSection(&_cffi_embed_startup_lock); +#endif +} + + +/********** CPython-specific section **********/ +#ifndef PYPY_VERSION + +#include "_cffi_errors.h" + + +#define _cffi_call_python_org _cffi_exports[_CFFI_CPIDX] + +PyMODINIT_FUNC _CFFI_PYTHON_STARTUP_FUNC(void); /* forward */ + +static void _cffi_py_initialize(void) +{ + /* XXX use initsigs=0, which "skips initialization registration of + signal handlers, which might be useful when Python is + embedded" according to the Python docs. But review and think + if it should be a user-controllable setting. + + XXX we should also give a way to write errors to a buffer + instead of to stderr. + + XXX if importing 'site' fails, CPython (any version) calls + exit(). Should we try to work around this behavior here? + */ + Py_InitializeEx(0); +} + +static int _cffi_initialize_python(void) +{ + /* This initializes Python, imports _cffi_backend, and then the + present .dll/.so is set up as a CPython C extension module. + */ + int result; + PyGILState_STATE state; + PyObject *pycode=NULL, *global_dict=NULL, *x; + PyObject *builtins; + + state = PyGILState_Ensure(); + + /* Call the initxxx() function from the present module. It will + create and initialize us as a CPython extension module, instead + of letting the startup Python code do it---it might reimport + the same .dll/.so and get maybe confused on some platforms. + It might also have troubles locating the .dll/.so again for all + I know. + */ + (void)_CFFI_PYTHON_STARTUP_FUNC(); + if (PyErr_Occurred()) + goto error; + + /* Now run the Python code provided to ffi.embedding_init_code(). + */ + pycode = Py_CompileString(_CFFI_PYTHON_STARTUP_CODE, + "", + Py_file_input); + if (pycode == NULL) + goto error; + global_dict = PyDict_New(); + if (global_dict == NULL) + goto error; + builtins = PyEval_GetBuiltins(); + if (builtins == NULL) + goto error; + if (PyDict_SetItemString(global_dict, "__builtins__", builtins) < 0) + goto error; + x = PyEval_EvalCode( +#if PY_MAJOR_VERSION < 3 + (PyCodeObject *) +#endif + pycode, global_dict, global_dict); + if (x == NULL) + goto error; + Py_DECREF(x); + + /* Done! Now if we've been called from + _cffi_start_and_call_python() in an ``extern "Python"``, we can + only hope that the Python code did correctly set up the + corresponding @ffi.def_extern() function. Otherwise, the + general logic of ``extern "Python"`` functions (inside the + _cffi_backend module) will find that the reference is still + missing and print an error. + */ + result = 0; + done: + Py_XDECREF(pycode); + Py_XDECREF(global_dict); + PyGILState_Release(state); + return result; + + error:; + { + /* Print as much information as potentially useful. + Debugging load-time failures with embedding is not fun + */ + PyObject *ecap; + PyObject *exception, *v, *tb, *f, *modules, *mod; + PyErr_Fetch(&exception, &v, &tb); + ecap = _cffi_start_error_capture(); + f = PySys_GetObject((char *)"stderr"); + if (f != NULL && f != Py_None) { + PyFile_WriteString( + "Failed to initialize the Python-CFFI embedding logic:\n\n", f); + } + + if (exception != NULL) { + PyErr_NormalizeException(&exception, &v, &tb); + PyErr_Display(exception, v, tb); + } + Py_XDECREF(exception); + Py_XDECREF(v); + Py_XDECREF(tb); + + if (f != NULL && f != Py_None) { + PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME + "\ncompiled with cffi version: 1.14.4" + "\n_cffi_backend module: ", f); + modules = PyImport_GetModuleDict(); + mod = PyDict_GetItemString(modules, "_cffi_backend"); + if (mod == NULL) { + PyFile_WriteString("not loaded", f); + } + else { + v = PyObject_GetAttrString(mod, "__file__"); + PyFile_WriteObject(v, f, 0); + Py_XDECREF(v); + } + PyFile_WriteString("\nsys.path: ", f); + PyFile_WriteObject(PySys_GetObject((char *)"path"), f, 0); + PyFile_WriteString("\n\n", f); + } + _cffi_stop_error_capture(ecap); + } + result = -1; + goto done; +} + +#if PY_VERSION_HEX < 0x03080000 +PyAPI_DATA(char *) _PyParser_TokenNames[]; /* from CPython */ +#endif + +static int _cffi_carefully_make_gil(void) +{ + /* This does the basic initialization of Python. It can be called + completely concurrently from unrelated threads. It assumes + that we don't hold the GIL before (if it exists), and we don't + hold it afterwards. + + (What it really does used to be completely different in Python 2 + and Python 3, with the Python 2 solution avoiding the spin-lock + around the Py_InitializeEx() call. However, after recent changes + to CPython 2.7 (issue #358) it no longer works. So we use the + Python 3 solution everywhere.) + + This initializes Python by calling Py_InitializeEx(). + Important: this must not be called concurrently at all. + So we use a global variable as a simple spin lock. This global + variable must be from 'libpythonX.Y.so', not from this + cffi-based extension module, because it must be shared from + different cffi-based extension modules. + + In Python < 3.8, we choose + _PyParser_TokenNames[0] as a completely arbitrary pointer value + that is never written to. The default is to point to the + string "ENDMARKER". We change it temporarily to point to the + next character in that string. (Yes, I know it's REALLY + obscure.) + + In Python >= 3.8, this string array is no longer writable, so + instead we pick PyCapsuleType.tp_version_tag. We can't change + Python < 3.8 because someone might use a mixture of cffi + embedded modules, some of which were compiled before this file + changed. + */ + +#ifdef WITH_THREAD +# if PY_VERSION_HEX < 0x03080000 + char *volatile *lock = (char *volatile *)_PyParser_TokenNames; + char *old_value, *locked_value; + + while (1) { /* spin loop */ + old_value = *lock; + locked_value = old_value + 1; + if (old_value[0] == 'E') { + assert(old_value[1] == 'N'); + if (cffi_compare_and_swap(lock, old_value, locked_value)) + break; + } + else { + assert(old_value[0] == 'N'); + /* should ideally do a spin loop instruction here, but + hard to do it portably and doesn't really matter I + think: PyEval_InitThreads() should be very fast, and + this is only run at start-up anyway. */ + } + } +# else + int volatile *lock = (int volatile *)&PyCapsule_Type.tp_version_tag; + int old_value, locked_value; + assert(!(PyCapsule_Type.tp_flags & Py_TPFLAGS_HAVE_VERSION_TAG)); + + while (1) { /* spin loop */ + old_value = *lock; + locked_value = -42; + if (old_value == 0) { + if (cffi_compare_and_swap(lock, old_value, locked_value)) + break; + } + else { + assert(old_value == locked_value); + /* should ideally do a spin loop instruction here, but + hard to do it portably and doesn't really matter I + think: PyEval_InitThreads() should be very fast, and + this is only run at start-up anyway. */ + } + } +# endif +#endif + + /* call Py_InitializeEx() */ + if (!Py_IsInitialized()) { + _cffi_py_initialize(); +#if PY_VERSION_HEX < 0x03070000 + PyEval_InitThreads(); +#endif + PyEval_SaveThread(); /* release the GIL */ + /* the returned tstate must be the one that has been stored into the + autoTLSkey by _PyGILState_Init() called from Py_Initialize(). */ + } + else { +#if PY_VERSION_HEX < 0x03070000 + /* PyEval_InitThreads() is always a no-op from CPython 3.7 */ + PyGILState_STATE state = PyGILState_Ensure(); + PyEval_InitThreads(); + PyGILState_Release(state); +#endif + } + +#ifdef WITH_THREAD + /* release the lock */ + while (!cffi_compare_and_swap(lock, locked_value, old_value)) + ; +#endif + + return 0; +} + +/********** end CPython-specific section **********/ + + +#else + + +/********** PyPy-specific section **********/ + +PyMODINIT_FUNC _CFFI_PYTHON_STARTUP_FUNC(const void *[]); /* forward */ + +static struct _cffi_pypy_init_s { + const char *name; + void *func; /* function pointer */ + const char *code; +} _cffi_pypy_init = { + _CFFI_MODULE_NAME, + _CFFI_PYTHON_STARTUP_FUNC, + _CFFI_PYTHON_STARTUP_CODE, +}; + +extern int pypy_carefully_make_gil(const char *); +extern int pypy_init_embedded_cffi_module(int, struct _cffi_pypy_init_s *); + +static int _cffi_carefully_make_gil(void) +{ + return pypy_carefully_make_gil(_CFFI_MODULE_NAME); +} + +static int _cffi_initialize_python(void) +{ + return pypy_init_embedded_cffi_module(0xB011, &_cffi_pypy_init); +} + +/********** end PyPy-specific section **********/ + + +#endif + + +#ifdef __GNUC__ +__attribute__((noinline)) +#endif +static _cffi_call_python_fnptr _cffi_start_python(void) +{ + /* Delicate logic to initialize Python. This function can be + called multiple times concurrently, e.g. when the process calls + its first ``extern "Python"`` functions in multiple threads at + once. It can also be called recursively, in which case we must + ignore it. We also have to consider what occurs if several + different cffi-based extensions reach this code in parallel + threads---it is a different copy of the code, then, and we + can't have any shared global variable unless it comes from + 'libpythonX.Y.so'. + + Idea: + + * _cffi_carefully_make_gil(): "carefully" call + PyEval_InitThreads() (possibly with Py_InitializeEx() first). + + * then we use a (local) custom lock to make sure that a call to this + cffi-based extension will wait if another call to the *same* + extension is running the initialization in another thread. + It is reentrant, so that a recursive call will not block, but + only one from a different thread. + + * then we grab the GIL and (Python 2) we call Py_InitializeEx(). + At this point, concurrent calls to Py_InitializeEx() are not + possible: we have the GIL. + + * do the rest of the specific initialization, which may + temporarily release the GIL but not the custom lock. + Only release the custom lock when we are done. + */ + static char called = 0; + + if (_cffi_carefully_make_gil() != 0) + return NULL; + + _cffi_acquire_reentrant_mutex(); + + /* Here the GIL exists, but we don't have it. We're only protected + from concurrency by the reentrant mutex. */ + + /* This file only initializes the embedded module once, the first + time this is called, even if there are subinterpreters. */ + if (!called) { + called = 1; /* invoke _cffi_initialize_python() only once, + but don't set '_cffi_call_python' right now, + otherwise concurrent threads won't call + this function at all (we need them to wait) */ + if (_cffi_initialize_python() == 0) { + /* now initialization is finished. Switch to the fast-path. */ + + /* We would like nobody to see the new value of + '_cffi_call_python' without also seeing the rest of the + data initialized. However, this is not possible. But + the new value of '_cffi_call_python' is the function + 'cffi_call_python()' from _cffi_backend. So: */ + cffi_write_barrier(); + /* ^^^ we put a write barrier here, and a corresponding + read barrier at the start of cffi_call_python(). This + ensures that after that read barrier, we see everything + done here before the write barrier. + */ + + assert(_cffi_call_python_org != NULL); + _cffi_call_python = (_cffi_call_python_fnptr)_cffi_call_python_org; + } + else { + /* initialization failed. Reset this to NULL, even if it was + already set to some other value. Future calls to + _cffi_start_python() are still forced to occur, and will + always return NULL from now on. */ + _cffi_call_python_org = NULL; + } + } + + _cffi_release_reentrant_mutex(); + + return (_cffi_call_python_fnptr)_cffi_call_python_org; +} + +static +void _cffi_start_and_call_python(struct _cffi_externpy_s *externpy, char *args) +{ + _cffi_call_python_fnptr fnptr; + int current_err = errno; +#ifdef _MSC_VER + int current_lasterr = GetLastError(); +#endif + fnptr = _cffi_start_python(); + if (fnptr == NULL) { + fprintf(stderr, "function %s() called, but initialization code " + "failed. Returning 0.\n", externpy->name); + memset(args, 0, externpy->size_of_result); + } +#ifdef _MSC_VER + SetLastError(current_lasterr); +#endif + errno = current_err; + + if (fnptr != NULL) + fnptr(externpy, args); +} + + +/* The cffi_start_python() function makes sure Python is initialized + and our cffi module is set up. It can be called manually from the + user C code. The same effect is obtained automatically from any + dll-exported ``extern "Python"`` function. This function returns + -1 if initialization failed, 0 if all is OK. */ +_CFFI_UNUSED_FN +static int cffi_start_python(void) +{ + if (_cffi_call_python == &_cffi_start_and_call_python) { + if (_cffi_start_python() == NULL) + return -1; + } + cffi_read_barrier(); + return 0; +} + +#undef cffi_compare_and_swap +#undef cffi_write_barrier +#undef cffi_read_barrier + +#ifdef __cplusplus +} +#endif diff --git a/WebCrawler/venv/Lib/site-packages/cffi/api.py b/WebCrawler/venv/Lib/site-packages/cffi/api.py new file mode 100644 index 0000000..999a8ae --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/cffi/api.py @@ -0,0 +1,965 @@ +import sys, types +from .lock import allocate_lock +from .error import CDefError +from . import model + +try: + callable +except NameError: + # Python 3.1 + from collections import Callable + callable = lambda x: isinstance(x, Callable) + +try: + basestring +except NameError: + # Python 3.x + basestring = str + +_unspecified = object() + + + +class FFI(object): + r''' + The main top-level class that you instantiate once, or once per module. + + Example usage: + + ffi = FFI() + ffi.cdef(""" + int printf(const char *, ...); + """) + + C = ffi.dlopen(None) # standard library + -or- + C = ffi.verify() # use a C compiler: verify the decl above is right + + C.printf("hello, %s!\n", ffi.new("char[]", "world")) + ''' + + def __init__(self, backend=None): + """Create an FFI instance. The 'backend' argument is used to + select a non-default backend, mostly for tests. + """ + if backend is None: + # You need PyPy (>= 2.0 beta), or a CPython (>= 2.6) with + # _cffi_backend.so compiled. + import _cffi_backend as backend + from . import __version__ + if backend.__version__ != __version__: + # bad version! Try to be as explicit as possible. + if hasattr(backend, '__file__'): + # CPython + raise Exception("Version mismatch: this is the 'cffi' package version %s, located in %r. When we import the top-level '_cffi_backend' extension module, we get version %s, located in %r. The two versions should be equal; check your installation." % ( + __version__, __file__, + backend.__version__, backend.__file__)) + else: + # PyPy + raise Exception("Version mismatch: this is the 'cffi' package version %s, located in %r. This interpreter comes with a built-in '_cffi_backend' module, which is version %s. The two versions should be equal; check your installation." % ( + __version__, __file__, backend.__version__)) + # (If you insist you can also try to pass the option + # 'backend=backend_ctypes.CTypesBackend()', but don't + # rely on it! It's probably not going to work well.) + + from . import cparser + self._backend = backend + self._lock = allocate_lock() + self._parser = cparser.Parser() + self._cached_btypes = {} + self._parsed_types = types.ModuleType('parsed_types').__dict__ + self._new_types = types.ModuleType('new_types').__dict__ + self._function_caches = [] + self._libraries = [] + self._cdefsources = [] + self._included_ffis = [] + self._windows_unicode = None + self._init_once_cache = {} + self._cdef_version = None + self._embedding = None + self._typecache = model.get_typecache(backend) + if hasattr(backend, 'set_ffi'): + backend.set_ffi(self) + for name in list(backend.__dict__): + if name.startswith('RTLD_'): + setattr(self, name, getattr(backend, name)) + # + with self._lock: + self.BVoidP = self._get_cached_btype(model.voidp_type) + self.BCharA = self._get_cached_btype(model.char_array_type) + if isinstance(backend, types.ModuleType): + # _cffi_backend: attach these constants to the class + if not hasattr(FFI, 'NULL'): + FFI.NULL = self.cast(self.BVoidP, 0) + FFI.CData, FFI.CType = backend._get_types() + else: + # ctypes backend: attach these constants to the instance + self.NULL = self.cast(self.BVoidP, 0) + self.CData, self.CType = backend._get_types() + self.buffer = backend.buffer + + def cdef(self, csource, override=False, packed=False, pack=None): + """Parse the given C source. This registers all declared functions, + types, and global variables. The functions and global variables can + then be accessed via either 'ffi.dlopen()' or 'ffi.verify()'. + The types can be used in 'ffi.new()' and other functions. + If 'packed' is specified as True, all structs declared inside this + cdef are packed, i.e. laid out without any field alignment at all. + Alternatively, 'pack' can be a small integer, and requests for + alignment greater than that are ignored (pack=1 is equivalent to + packed=True). + """ + self._cdef(csource, override=override, packed=packed, pack=pack) + + def embedding_api(self, csource, packed=False, pack=None): + self._cdef(csource, packed=packed, pack=pack, dllexport=True) + if self._embedding is None: + self._embedding = '' + + def _cdef(self, csource, override=False, **options): + if not isinstance(csource, str): # unicode, on Python 2 + if not isinstance(csource, basestring): + raise TypeError("cdef() argument must be a string") + csource = csource.encode('ascii') + with self._lock: + self._cdef_version = object() + self._parser.parse(csource, override=override, **options) + self._cdefsources.append(csource) + if override: + for cache in self._function_caches: + cache.clear() + finishlist = self._parser._recomplete + if finishlist: + self._parser._recomplete = [] + for tp in finishlist: + tp.finish_backend_type(self, finishlist) + + def dlopen(self, name, flags=0): + """Load and return a dynamic library identified by 'name'. + The standard C library can be loaded by passing None. + Note that functions and types declared by 'ffi.cdef()' are not + linked to a particular library, just like C headers; in the + library we only look for the actual (untyped) symbols. + """ + if not (isinstance(name, basestring) or + name is None or + isinstance(name, self.CData)): + raise TypeError("dlopen(name): name must be a file name, None, " + "or an already-opened 'void *' handle") + with self._lock: + lib, function_cache = _make_ffi_library(self, name, flags) + self._function_caches.append(function_cache) + self._libraries.append(lib) + return lib + + def dlclose(self, lib): + """Close a library obtained with ffi.dlopen(). After this call, + access to functions or variables from the library will fail + (possibly with a segmentation fault). + """ + type(lib).__cffi_close__(lib) + + def _typeof_locked(self, cdecl): + # call me with the lock! + key = cdecl + if key in self._parsed_types: + return self._parsed_types[key] + # + if not isinstance(cdecl, str): # unicode, on Python 2 + cdecl = cdecl.encode('ascii') + # + type = self._parser.parse_type(cdecl) + really_a_function_type = type.is_raw_function + if really_a_function_type: + type = type.as_function_pointer() + btype = self._get_cached_btype(type) + result = btype, really_a_function_type + self._parsed_types[key] = result + return result + + def _typeof(self, cdecl, consider_function_as_funcptr=False): + # string -> ctype object + try: + result = self._parsed_types[cdecl] + except KeyError: + with self._lock: + result = self._typeof_locked(cdecl) + # + btype, really_a_function_type = result + if really_a_function_type and not consider_function_as_funcptr: + raise CDefError("the type %r is a function type, not a " + "pointer-to-function type" % (cdecl,)) + return btype + + def typeof(self, cdecl): + """Parse the C type given as a string and return the + corresponding object. + It can also be used on 'cdata' instance to get its C type. + """ + if isinstance(cdecl, basestring): + return self._typeof(cdecl) + if isinstance(cdecl, self.CData): + return self._backend.typeof(cdecl) + if isinstance(cdecl, types.BuiltinFunctionType): + res = _builtin_function_type(cdecl) + if res is not None: + return res + if (isinstance(cdecl, types.FunctionType) + and hasattr(cdecl, '_cffi_base_type')): + with self._lock: + return self._get_cached_btype(cdecl._cffi_base_type) + raise TypeError(type(cdecl)) + + def sizeof(self, cdecl): + """Return the size in bytes of the argument. It can be a + string naming a C type, or a 'cdata' instance. + """ + if isinstance(cdecl, basestring): + BType = self._typeof(cdecl) + return self._backend.sizeof(BType) + else: + return self._backend.sizeof(cdecl) + + def alignof(self, cdecl): + """Return the natural alignment size in bytes of the C type + given as a string. + """ + if isinstance(cdecl, basestring): + cdecl = self._typeof(cdecl) + return self._backend.alignof(cdecl) + + def offsetof(self, cdecl, *fields_or_indexes): + """Return the offset of the named field inside the given + structure or array, which must be given as a C type name. + You can give several field names in case of nested structures. + You can also give numeric values which correspond to array + items, in case of an array type. + """ + if isinstance(cdecl, basestring): + cdecl = self._typeof(cdecl) + return self._typeoffsetof(cdecl, *fields_or_indexes)[1] + + def new(self, cdecl, init=None): + """Allocate an instance according to the specified C type and + return a pointer to it. The specified C type must be either a + pointer or an array: ``new('X *')`` allocates an X and returns + a pointer to it, whereas ``new('X[n]')`` allocates an array of + n X'es and returns an array referencing it (which works + mostly like a pointer, like in C). You can also use + ``new('X[]', n)`` to allocate an array of a non-constant + length n. + + The memory is initialized following the rules of declaring a + global variable in C: by default it is zero-initialized, but + an explicit initializer can be given which can be used to + fill all or part of the memory. + + When the returned object goes out of scope, the memory + is freed. In other words the returned object has + ownership of the value of type 'cdecl' that it points to. This + means that the raw data can be used as long as this object is + kept alive, but must not be used for a longer time. Be careful + about that when copying the pointer to the memory somewhere + else, e.g. into another structure. + """ + if isinstance(cdecl, basestring): + cdecl = self._typeof(cdecl) + return self._backend.newp(cdecl, init) + + def new_allocator(self, alloc=None, free=None, + should_clear_after_alloc=True): + """Return a new allocator, i.e. a function that behaves like ffi.new() + but uses the provided low-level 'alloc' and 'free' functions. + + 'alloc' is called with the size as argument. If it returns NULL, a + MemoryError is raised. 'free' is called with the result of 'alloc' + as argument. Both can be either Python function or directly C + functions. If 'free' is None, then no free function is called. + If both 'alloc' and 'free' are None, the default is used. + + If 'should_clear_after_alloc' is set to False, then the memory + returned by 'alloc' is assumed to be already cleared (or you are + fine with garbage); otherwise CFFI will clear it. + """ + compiled_ffi = self._backend.FFI() + allocator = compiled_ffi.new_allocator(alloc, free, + should_clear_after_alloc) + def allocate(cdecl, init=None): + if isinstance(cdecl, basestring): + cdecl = self._typeof(cdecl) + return allocator(cdecl, init) + return allocate + + def cast(self, cdecl, source): + """Similar to a C cast: returns an instance of the named C + type initialized with the given 'source'. The source is + casted between integers or pointers of any type. + """ + if isinstance(cdecl, basestring): + cdecl = self._typeof(cdecl) + return self._backend.cast(cdecl, source) + + def string(self, cdata, maxlen=-1): + """Return a Python string (or unicode string) from the 'cdata'. + If 'cdata' is a pointer or array of characters or bytes, returns + the null-terminated string. The returned string extends until + the first null character, or at most 'maxlen' characters. If + 'cdata' is an array then 'maxlen' defaults to its length. + + If 'cdata' is a pointer or array of wchar_t, returns a unicode + string following the same rules. + + If 'cdata' is a single character or byte or a wchar_t, returns + it as a string or unicode string. + + If 'cdata' is an enum, returns the value of the enumerator as a + string, or 'NUMBER' if the value is out of range. + """ + return self._backend.string(cdata, maxlen) + + def unpack(self, cdata, length): + """Unpack an array of C data of the given length, + returning a Python string/unicode/list. + + If 'cdata' is a pointer to 'char', returns a byte string. + It does not stop at the first null. This is equivalent to: + ffi.buffer(cdata, length)[:] + + If 'cdata' is a pointer to 'wchar_t', returns a unicode string. + 'length' is measured in wchar_t's; it is not the size in bytes. + + If 'cdata' is a pointer to anything else, returns a list of + 'length' items. This is a faster equivalent to: + [cdata[i] for i in range(length)] + """ + return self._backend.unpack(cdata, length) + + #def buffer(self, cdata, size=-1): + # """Return a read-write buffer object that references the raw C data + # pointed to by the given 'cdata'. The 'cdata' must be a pointer or + # an array. Can be passed to functions expecting a buffer, or directly + # manipulated with: + # + # buf[:] get a copy of it in a regular string, or + # buf[idx] as a single character + # buf[:] = ... + # buf[idx] = ... change the content + # """ + # note that 'buffer' is a type, set on this instance by __init__ + + def from_buffer(self, cdecl, python_buffer=_unspecified, + require_writable=False): + """Return a cdata of the given type pointing to the data of the + given Python object, which must support the buffer interface. + Note that this is not meant to be used on the built-in types + str or unicode (you can build 'char[]' arrays explicitly) + but only on objects containing large quantities of raw data + in some other format, like 'array.array' or numpy arrays. + + The first argument is optional and default to 'char[]'. + """ + if python_buffer is _unspecified: + cdecl, python_buffer = self.BCharA, cdecl + elif isinstance(cdecl, basestring): + cdecl = self._typeof(cdecl) + return self._backend.from_buffer(cdecl, python_buffer, + require_writable) + + def memmove(self, dest, src, n): + """ffi.memmove(dest, src, n) copies n bytes of memory from src to dest. + + Like the C function memmove(), the memory areas may overlap; + apart from that it behaves like the C function memcpy(). + + 'src' can be any cdata ptr or array, or any Python buffer object. + 'dest' can be any cdata ptr or array, or a writable Python buffer + object. The size to copy, 'n', is always measured in bytes. + + Unlike other methods, this one supports all Python buffer including + byte strings and bytearrays---but it still does not support + non-contiguous buffers. + """ + return self._backend.memmove(dest, src, n) + + def callback(self, cdecl, python_callable=None, error=None, onerror=None): + """Return a callback object or a decorator making such a + callback object. 'cdecl' must name a C function pointer type. + The callback invokes the specified 'python_callable' (which may + be provided either directly or via a decorator). Important: the + callback object must be manually kept alive for as long as the + callback may be invoked from the C level. + """ + def callback_decorator_wrap(python_callable): + if not callable(python_callable): + raise TypeError("the 'python_callable' argument " + "is not callable") + return self._backend.callback(cdecl, python_callable, + error, onerror) + if isinstance(cdecl, basestring): + cdecl = self._typeof(cdecl, consider_function_as_funcptr=True) + if python_callable is None: + return callback_decorator_wrap # decorator mode + else: + return callback_decorator_wrap(python_callable) # direct mode + + def getctype(self, cdecl, replace_with=''): + """Return a string giving the C type 'cdecl', which may be itself + a string or a object. If 'replace_with' is given, it gives + extra text to append (or insert for more complicated C types), like + a variable name, or '*' to get actually the C type 'pointer-to-cdecl'. + """ + if isinstance(cdecl, basestring): + cdecl = self._typeof(cdecl) + replace_with = replace_with.strip() + if (replace_with.startswith('*') + and '&[' in self._backend.getcname(cdecl, '&')): + replace_with = '(%s)' % replace_with + elif replace_with and not replace_with[0] in '[(': + replace_with = ' ' + replace_with + return self._backend.getcname(cdecl, replace_with) + + def gc(self, cdata, destructor, size=0): + """Return a new cdata object that points to the same + data. Later, when this new cdata object is garbage-collected, + 'destructor(old_cdata_object)' will be called. + + The optional 'size' gives an estimate of the size, used to + trigger the garbage collection more eagerly. So far only used + on PyPy. It tells the GC that the returned object keeps alive + roughly 'size' bytes of external memory. + """ + return self._backend.gcp(cdata, destructor, size) + + def _get_cached_btype(self, type): + assert self._lock.acquire(False) is False + # call me with the lock! + try: + BType = self._cached_btypes[type] + except KeyError: + finishlist = [] + BType = type.get_cached_btype(self, finishlist) + for type in finishlist: + type.finish_backend_type(self, finishlist) + return BType + + def verify(self, source='', tmpdir=None, **kwargs): + """Verify that the current ffi signatures compile on this + machine, and return a dynamic library object. The dynamic + library can be used to call functions and access global + variables declared in this 'ffi'. The library is compiled + by the C compiler: it gives you C-level API compatibility + (including calling macros). This is unlike 'ffi.dlopen()', + which requires binary compatibility in the signatures. + """ + from .verifier import Verifier, _caller_dir_pycache + # + # If set_unicode(True) was called, insert the UNICODE and + # _UNICODE macro declarations + if self._windows_unicode: + self._apply_windows_unicode(kwargs) + # + # Set the tmpdir here, and not in Verifier.__init__: it picks + # up the caller's directory, which we want to be the caller of + # ffi.verify(), as opposed to the caller of Veritier(). + tmpdir = tmpdir or _caller_dir_pycache() + # + # Make a Verifier() and use it to load the library. + self.verifier = Verifier(self, source, tmpdir, **kwargs) + lib = self.verifier.load_library() + # + # Save the loaded library for keep-alive purposes, even + # if the caller doesn't keep it alive itself (it should). + self._libraries.append(lib) + return lib + + def _get_errno(self): + return self._backend.get_errno() + def _set_errno(self, errno): + self._backend.set_errno(errno) + errno = property(_get_errno, _set_errno, None, + "the value of 'errno' from/to the C calls") + + def getwinerror(self, code=-1): + return self._backend.getwinerror(code) + + def _pointer_to(self, ctype): + with self._lock: + return model.pointer_cache(self, ctype) + + def addressof(self, cdata, *fields_or_indexes): + """Return the address of a . + If 'fields_or_indexes' are given, returns the address of that + field or array item in the structure or array, recursively in + case of nested structures. + """ + try: + ctype = self._backend.typeof(cdata) + except TypeError: + if '__addressof__' in type(cdata).__dict__: + return type(cdata).__addressof__(cdata, *fields_or_indexes) + raise + if fields_or_indexes: + ctype, offset = self._typeoffsetof(ctype, *fields_or_indexes) + else: + if ctype.kind == "pointer": + raise TypeError("addressof(pointer)") + offset = 0 + ctypeptr = self._pointer_to(ctype) + return self._backend.rawaddressof(ctypeptr, cdata, offset) + + def _typeoffsetof(self, ctype, field_or_index, *fields_or_indexes): + ctype, offset = self._backend.typeoffsetof(ctype, field_or_index) + for field1 in fields_or_indexes: + ctype, offset1 = self._backend.typeoffsetof(ctype, field1, 1) + offset += offset1 + return ctype, offset + + def include(self, ffi_to_include): + """Includes the typedefs, structs, unions and enums defined + in another FFI instance. Usage is similar to a #include in C, + where a part of the program might include types defined in + another part for its own usage. Note that the include() + method has no effect on functions, constants and global + variables, which must anyway be accessed directly from the + lib object returned by the original FFI instance. + """ + if not isinstance(ffi_to_include, FFI): + raise TypeError("ffi.include() expects an argument that is also of" + " type cffi.FFI, not %r" % ( + type(ffi_to_include).__name__,)) + if ffi_to_include is self: + raise ValueError("self.include(self)") + with ffi_to_include._lock: + with self._lock: + self._parser.include(ffi_to_include._parser) + self._cdefsources.append('[') + self._cdefsources.extend(ffi_to_include._cdefsources) + self._cdefsources.append(']') + self._included_ffis.append(ffi_to_include) + + def new_handle(self, x): + return self._backend.newp_handle(self.BVoidP, x) + + def from_handle(self, x): + return self._backend.from_handle(x) + + def release(self, x): + self._backend.release(x) + + def set_unicode(self, enabled_flag): + """Windows: if 'enabled_flag' is True, enable the UNICODE and + _UNICODE defines in C, and declare the types like TCHAR and LPTCSTR + to be (pointers to) wchar_t. If 'enabled_flag' is False, + declare these types to be (pointers to) plain 8-bit characters. + This is mostly for backward compatibility; you usually want True. + """ + if self._windows_unicode is not None: + raise ValueError("set_unicode() can only be called once") + enabled_flag = bool(enabled_flag) + if enabled_flag: + self.cdef("typedef wchar_t TBYTE;" + "typedef wchar_t TCHAR;" + "typedef const wchar_t *LPCTSTR;" + "typedef const wchar_t *PCTSTR;" + "typedef wchar_t *LPTSTR;" + "typedef wchar_t *PTSTR;" + "typedef TBYTE *PTBYTE;" + "typedef TCHAR *PTCHAR;") + else: + self.cdef("typedef char TBYTE;" + "typedef char TCHAR;" + "typedef const char *LPCTSTR;" + "typedef const char *PCTSTR;" + "typedef char *LPTSTR;" + "typedef char *PTSTR;" + "typedef TBYTE *PTBYTE;" + "typedef TCHAR *PTCHAR;") + self._windows_unicode = enabled_flag + + def _apply_windows_unicode(self, kwds): + defmacros = kwds.get('define_macros', ()) + if not isinstance(defmacros, (list, tuple)): + raise TypeError("'define_macros' must be a list or tuple") + defmacros = list(defmacros) + [('UNICODE', '1'), + ('_UNICODE', '1')] + kwds['define_macros'] = defmacros + + def _apply_embedding_fix(self, kwds): + # must include an argument like "-lpython2.7" for the compiler + def ensure(key, value): + lst = kwds.setdefault(key, []) + if value not in lst: + lst.append(value) + # + if '__pypy__' in sys.builtin_module_names: + import os + if sys.platform == "win32": + # we need 'libpypy-c.lib'. Current distributions of + # pypy (>= 4.1) contain it as 'libs/python27.lib'. + pythonlib = "python{0[0]}{0[1]}".format(sys.version_info) + if hasattr(sys, 'prefix'): + ensure('library_dirs', os.path.join(sys.prefix, 'libs')) + else: + # we need 'libpypy-c.{so,dylib}', which should be by + # default located in 'sys.prefix/bin' for installed + # systems. + if sys.version_info < (3,): + pythonlib = "pypy-c" + else: + pythonlib = "pypy3-c" + if hasattr(sys, 'prefix'): + ensure('library_dirs', os.path.join(sys.prefix, 'bin')) + # On uninstalled pypy's, the libpypy-c is typically found in + # .../pypy/goal/. + if hasattr(sys, 'prefix'): + ensure('library_dirs', os.path.join(sys.prefix, 'pypy', 'goal')) + else: + if sys.platform == "win32": + template = "python%d%d" + if hasattr(sys, 'gettotalrefcount'): + template += '_d' + else: + try: + import sysconfig + except ImportError: # 2.6 + from distutils import sysconfig + template = "python%d.%d" + if sysconfig.get_config_var('DEBUG_EXT'): + template += sysconfig.get_config_var('DEBUG_EXT') + pythonlib = (template % + (sys.hexversion >> 24, (sys.hexversion >> 16) & 0xff)) + if hasattr(sys, 'abiflags'): + pythonlib += sys.abiflags + ensure('libraries', pythonlib) + if sys.platform == "win32": + ensure('extra_link_args', '/MANIFEST') + + def set_source(self, module_name, source, source_extension='.c', **kwds): + import os + if hasattr(self, '_assigned_source'): + raise ValueError("set_source() cannot be called several times " + "per ffi object") + if not isinstance(module_name, basestring): + raise TypeError("'module_name' must be a string") + if os.sep in module_name or (os.altsep and os.altsep in module_name): + raise ValueError("'module_name' must not contain '/': use a dotted " + "name to make a 'package.module' location") + self._assigned_source = (str(module_name), source, + source_extension, kwds) + + def set_source_pkgconfig(self, module_name, pkgconfig_libs, source, + source_extension='.c', **kwds): + from . import pkgconfig + if not isinstance(pkgconfig_libs, list): + raise TypeError("the pkgconfig_libs argument must be a list " + "of package names") + kwds2 = pkgconfig.flags_from_pkgconfig(pkgconfig_libs) + pkgconfig.merge_flags(kwds, kwds2) + self.set_source(module_name, source, source_extension, **kwds) + + def distutils_extension(self, tmpdir='build', verbose=True): + from distutils.dir_util import mkpath + from .recompiler import recompile + # + if not hasattr(self, '_assigned_source'): + if hasattr(self, 'verifier'): # fallback, 'tmpdir' ignored + return self.verifier.get_extension() + raise ValueError("set_source() must be called before" + " distutils_extension()") + module_name, source, source_extension, kwds = self._assigned_source + if source is None: + raise TypeError("distutils_extension() is only for C extension " + "modules, not for dlopen()-style pure Python " + "modules") + mkpath(tmpdir) + ext, updated = recompile(self, module_name, + source, tmpdir=tmpdir, extradir=tmpdir, + source_extension=source_extension, + call_c_compiler=False, **kwds) + if verbose: + if updated: + sys.stderr.write("regenerated: %r\n" % (ext.sources[0],)) + else: + sys.stderr.write("not modified: %r\n" % (ext.sources[0],)) + return ext + + def emit_c_code(self, filename): + from .recompiler import recompile + # + if not hasattr(self, '_assigned_source'): + raise ValueError("set_source() must be called before emit_c_code()") + module_name, source, source_extension, kwds = self._assigned_source + if source is None: + raise TypeError("emit_c_code() is only for C extension modules, " + "not for dlopen()-style pure Python modules") + recompile(self, module_name, source, + c_file=filename, call_c_compiler=False, **kwds) + + def emit_python_code(self, filename): + from .recompiler import recompile + # + if not hasattr(self, '_assigned_source'): + raise ValueError("set_source() must be called before emit_c_code()") + module_name, source, source_extension, kwds = self._assigned_source + if source is not None: + raise TypeError("emit_python_code() is only for dlopen()-style " + "pure Python modules, not for C extension modules") + recompile(self, module_name, source, + c_file=filename, call_c_compiler=False, **kwds) + + def compile(self, tmpdir='.', verbose=0, target=None, debug=None): + """The 'target' argument gives the final file name of the + compiled DLL. Use '*' to force distutils' choice, suitable for + regular CPython C API modules. Use a file name ending in '.*' + to ask for the system's default extension for dynamic libraries + (.so/.dll/.dylib). + + The default is '*' when building a non-embedded C API extension, + and (module_name + '.*') when building an embedded library. + """ + from .recompiler import recompile + # + if not hasattr(self, '_assigned_source'): + raise ValueError("set_source() must be called before compile()") + module_name, source, source_extension, kwds = self._assigned_source + return recompile(self, module_name, source, tmpdir=tmpdir, + target=target, source_extension=source_extension, + compiler_verbose=verbose, debug=debug, **kwds) + + def init_once(self, func, tag): + # Read _init_once_cache[tag], which is either (False, lock) if + # we're calling the function now in some thread, or (True, result). + # Don't call setdefault() in most cases, to avoid allocating and + # immediately freeing a lock; but still use setdefaut() to avoid + # races. + try: + x = self._init_once_cache[tag] + except KeyError: + x = self._init_once_cache.setdefault(tag, (False, allocate_lock())) + # Common case: we got (True, result), so we return the result. + if x[0]: + return x[1] + # Else, it's a lock. Acquire it to serialize the following tests. + with x[1]: + # Read again from _init_once_cache the current status. + x = self._init_once_cache[tag] + if x[0]: + return x[1] + # Call the function and store the result back. + result = func() + self._init_once_cache[tag] = (True, result) + return result + + def embedding_init_code(self, pysource): + if self._embedding: + raise ValueError("embedding_init_code() can only be called once") + # fix 'pysource' before it gets dumped into the C file: + # - remove empty lines at the beginning, so it starts at "line 1" + # - dedent, if all non-empty lines are indented + # - check for SyntaxErrors + import re + match = re.match(r'\s*\n', pysource) + if match: + pysource = pysource[match.end():] + lines = pysource.splitlines() or [''] + prefix = re.match(r'\s*', lines[0]).group() + for i in range(1, len(lines)): + line = lines[i] + if line.rstrip(): + while not line.startswith(prefix): + prefix = prefix[:-1] + i = len(prefix) + lines = [line[i:]+'\n' for line in lines] + pysource = ''.join(lines) + # + compile(pysource, "cffi_init", "exec") + # + self._embedding = pysource + + def def_extern(self, *args, **kwds): + raise ValueError("ffi.def_extern() is only available on API-mode FFI " + "objects") + + def list_types(self): + """Returns the user type names known to this FFI instance. + This returns a tuple containing three lists of names: + (typedef_names, names_of_structs, names_of_unions) + """ + typedefs = [] + structs = [] + unions = [] + for key in self._parser._declarations: + if key.startswith('typedef '): + typedefs.append(key[8:]) + elif key.startswith('struct '): + structs.append(key[7:]) + elif key.startswith('union '): + unions.append(key[6:]) + typedefs.sort() + structs.sort() + unions.sort() + return (typedefs, structs, unions) + + +def _load_backend_lib(backend, name, flags): + import os + if not isinstance(name, basestring): + if sys.platform != "win32" or name is not None: + return backend.load_library(name, flags) + name = "c" # Windows: load_library(None) fails, but this works + # on Python 2 (backward compatibility hack only) + first_error = None + if '.' in name or '/' in name or os.sep in name: + try: + return backend.load_library(name, flags) + except OSError as e: + first_error = e + import ctypes.util + path = ctypes.util.find_library(name) + if path is None: + if name == "c" and sys.platform == "win32" and sys.version_info >= (3,): + raise OSError("dlopen(None) cannot work on Windows for Python 3 " + "(see http://bugs.python.org/issue23606)") + msg = ("ctypes.util.find_library() did not manage " + "to locate a library called %r" % (name,)) + if first_error is not None: + msg = "%s. Additionally, %s" % (first_error, msg) + raise OSError(msg) + return backend.load_library(path, flags) + +def _make_ffi_library(ffi, libname, flags): + backend = ffi._backend + backendlib = _load_backend_lib(backend, libname, flags) + # + def accessor_function(name): + key = 'function ' + name + tp, _ = ffi._parser._declarations[key] + BType = ffi._get_cached_btype(tp) + value = backendlib.load_function(BType, name) + library.__dict__[name] = value + # + def accessor_variable(name): + key = 'variable ' + name + tp, _ = ffi._parser._declarations[key] + BType = ffi._get_cached_btype(tp) + read_variable = backendlib.read_variable + write_variable = backendlib.write_variable + setattr(FFILibrary, name, property( + lambda self: read_variable(BType, name), + lambda self, value: write_variable(BType, name, value))) + # + def addressof_var(name): + try: + return addr_variables[name] + except KeyError: + with ffi._lock: + if name not in addr_variables: + key = 'variable ' + name + tp, _ = ffi._parser._declarations[key] + BType = ffi._get_cached_btype(tp) + if BType.kind != 'array': + BType = model.pointer_cache(ffi, BType) + p = backendlib.load_function(BType, name) + addr_variables[name] = p + return addr_variables[name] + # + def accessor_constant(name): + raise NotImplementedError("non-integer constant '%s' cannot be " + "accessed from a dlopen() library" % (name,)) + # + def accessor_int_constant(name): + library.__dict__[name] = ffi._parser._int_constants[name] + # + accessors = {} + accessors_version = [False] + addr_variables = {} + # + def update_accessors(): + if accessors_version[0] is ffi._cdef_version: + return + # + for key, (tp, _) in ffi._parser._declarations.items(): + if not isinstance(tp, model.EnumType): + tag, name = key.split(' ', 1) + if tag == 'function': + accessors[name] = accessor_function + elif tag == 'variable': + accessors[name] = accessor_variable + elif tag == 'constant': + accessors[name] = accessor_constant + else: + for i, enumname in enumerate(tp.enumerators): + def accessor_enum(name, tp=tp, i=i): + tp.check_not_partial() + library.__dict__[name] = tp.enumvalues[i] + accessors[enumname] = accessor_enum + for name in ffi._parser._int_constants: + accessors.setdefault(name, accessor_int_constant) + accessors_version[0] = ffi._cdef_version + # + def make_accessor(name): + with ffi._lock: + if name in library.__dict__ or name in FFILibrary.__dict__: + return # added by another thread while waiting for the lock + if name not in accessors: + update_accessors() + if name not in accessors: + raise AttributeError(name) + accessors[name](name) + # + class FFILibrary(object): + def __getattr__(self, name): + make_accessor(name) + return getattr(self, name) + def __setattr__(self, name, value): + try: + property = getattr(self.__class__, name) + except AttributeError: + make_accessor(name) + setattr(self, name, value) + else: + property.__set__(self, value) + def __dir__(self): + with ffi._lock: + update_accessors() + return accessors.keys() + def __addressof__(self, name): + if name in library.__dict__: + return library.__dict__[name] + if name in FFILibrary.__dict__: + return addressof_var(name) + make_accessor(name) + if name in library.__dict__: + return library.__dict__[name] + if name in FFILibrary.__dict__: + return addressof_var(name) + raise AttributeError("cffi library has no function or " + "global variable named '%s'" % (name,)) + def __cffi_close__(self): + backendlib.close_lib() + self.__dict__.clear() + # + if isinstance(libname, basestring): + try: + if not isinstance(libname, str): # unicode, on Python 2 + libname = libname.encode('utf-8') + FFILibrary.__name__ = 'FFILibrary_%s' % libname + except UnicodeError: + pass + library = FFILibrary() + return library, library.__dict__ + +def _builtin_function_type(func): + # a hack to make at least ffi.typeof(builtin_function) work, + # if the builtin function was obtained by 'vengine_cpy'. + import sys + try: + module = sys.modules[func.__module__] + ffi = module._cffi_original_ffi + types_of_builtin_funcs = module._cffi_types_of_builtin_funcs + tp = types_of_builtin_funcs[func] + except (KeyError, AttributeError, TypeError): + return None + else: + with ffi._lock: + return ffi._get_cached_btype(tp) diff --git a/WebCrawler/venv/Lib/site-packages/cffi/backend_ctypes.py b/WebCrawler/venv/Lib/site-packages/cffi/backend_ctypes.py new file mode 100644 index 0000000..e7956a7 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/cffi/backend_ctypes.py @@ -0,0 +1,1121 @@ +import ctypes, ctypes.util, operator, sys +from . import model + +if sys.version_info < (3,): + bytechr = chr +else: + unicode = str + long = int + xrange = range + bytechr = lambda num: bytes([num]) + +class CTypesType(type): + pass + +class CTypesData(object): + __metaclass__ = CTypesType + __slots__ = ['__weakref__'] + __name__ = '' + + def __init__(self, *args): + raise TypeError("cannot instantiate %r" % (self.__class__,)) + + @classmethod + def _newp(cls, init): + raise TypeError("expected a pointer or array ctype, got '%s'" + % (cls._get_c_name(),)) + + @staticmethod + def _to_ctypes(value): + raise TypeError + + @classmethod + def _arg_to_ctypes(cls, *value): + try: + ctype = cls._ctype + except AttributeError: + raise TypeError("cannot create an instance of %r" % (cls,)) + if value: + res = cls._to_ctypes(*value) + if not isinstance(res, ctype): + res = cls._ctype(res) + else: + res = cls._ctype() + return res + + @classmethod + def _create_ctype_obj(cls, init): + if init is None: + return cls._arg_to_ctypes() + else: + return cls._arg_to_ctypes(init) + + @staticmethod + def _from_ctypes(ctypes_value): + raise TypeError + + @classmethod + def _get_c_name(cls, replace_with=''): + return cls._reftypename.replace(' &', replace_with) + + @classmethod + def _fix_class(cls): + cls.__name__ = 'CData<%s>' % (cls._get_c_name(),) + cls.__qualname__ = 'CData<%s>' % (cls._get_c_name(),) + cls.__module__ = 'ffi' + + def _get_own_repr(self): + raise NotImplementedError + + def _addr_repr(self, address): + if address == 0: + return 'NULL' + else: + if address < 0: + address += 1 << (8*ctypes.sizeof(ctypes.c_void_p)) + return '0x%x' % address + + def __repr__(self, c_name=None): + own = self._get_own_repr() + return '' % (c_name or self._get_c_name(), own) + + def _convert_to_address(self, BClass): + if BClass is None: + raise TypeError("cannot convert %r to an address" % ( + self._get_c_name(),)) + else: + raise TypeError("cannot convert %r to %r" % ( + self._get_c_name(), BClass._get_c_name())) + + @classmethod + def _get_size(cls): + return ctypes.sizeof(cls._ctype) + + def _get_size_of_instance(self): + return ctypes.sizeof(self._ctype) + + @classmethod + def _cast_from(cls, source): + raise TypeError("cannot cast to %r" % (cls._get_c_name(),)) + + def _cast_to_integer(self): + return self._convert_to_address(None) + + @classmethod + def _alignment(cls): + return ctypes.alignment(cls._ctype) + + def __iter__(self): + raise TypeError("cdata %r does not support iteration" % ( + self._get_c_name()),) + + def _make_cmp(name): + cmpfunc = getattr(operator, name) + def cmp(self, other): + v_is_ptr = not isinstance(self, CTypesGenericPrimitive) + w_is_ptr = (isinstance(other, CTypesData) and + not isinstance(other, CTypesGenericPrimitive)) + if v_is_ptr and w_is_ptr: + return cmpfunc(self._convert_to_address(None), + other._convert_to_address(None)) + elif v_is_ptr or w_is_ptr: + return NotImplemented + else: + if isinstance(self, CTypesGenericPrimitive): + self = self._value + if isinstance(other, CTypesGenericPrimitive): + other = other._value + return cmpfunc(self, other) + cmp.func_name = name + return cmp + + __eq__ = _make_cmp('__eq__') + __ne__ = _make_cmp('__ne__') + __lt__ = _make_cmp('__lt__') + __le__ = _make_cmp('__le__') + __gt__ = _make_cmp('__gt__') + __ge__ = _make_cmp('__ge__') + + def __hash__(self): + return hash(self._convert_to_address(None)) + + def _to_string(self, maxlen): + raise TypeError("string(): %r" % (self,)) + + +class CTypesGenericPrimitive(CTypesData): + __slots__ = [] + + def __hash__(self): + return hash(self._value) + + def _get_own_repr(self): + return repr(self._from_ctypes(self._value)) + + +class CTypesGenericArray(CTypesData): + __slots__ = [] + + @classmethod + def _newp(cls, init): + return cls(init) + + def __iter__(self): + for i in xrange(len(self)): + yield self[i] + + def _get_own_repr(self): + return self._addr_repr(ctypes.addressof(self._blob)) + + +class CTypesGenericPtr(CTypesData): + __slots__ = ['_address', '_as_ctype_ptr'] + _automatic_casts = False + kind = "pointer" + + @classmethod + def _newp(cls, init): + return cls(init) + + @classmethod + def _cast_from(cls, source): + if source is None: + address = 0 + elif isinstance(source, CTypesData): + address = source._cast_to_integer() + elif isinstance(source, (int, long)): + address = source + else: + raise TypeError("bad type for cast to %r: %r" % + (cls, type(source).__name__)) + return cls._new_pointer_at(address) + + @classmethod + def _new_pointer_at(cls, address): + self = cls.__new__(cls) + self._address = address + self._as_ctype_ptr = ctypes.cast(address, cls._ctype) + return self + + def _get_own_repr(self): + try: + return self._addr_repr(self._address) + except AttributeError: + return '???' + + def _cast_to_integer(self): + return self._address + + def __nonzero__(self): + return bool(self._address) + __bool__ = __nonzero__ + + @classmethod + def _to_ctypes(cls, value): + if not isinstance(value, CTypesData): + raise TypeError("unexpected %s object" % type(value).__name__) + address = value._convert_to_address(cls) + return ctypes.cast(address, cls._ctype) + + @classmethod + def _from_ctypes(cls, ctypes_ptr): + address = ctypes.cast(ctypes_ptr, ctypes.c_void_p).value or 0 + return cls._new_pointer_at(address) + + @classmethod + def _initialize(cls, ctypes_ptr, value): + if value: + ctypes_ptr.contents = cls._to_ctypes(value).contents + + def _convert_to_address(self, BClass): + if (BClass in (self.__class__, None) or BClass._automatic_casts + or self._automatic_casts): + return self._address + else: + return CTypesData._convert_to_address(self, BClass) + + +class CTypesBaseStructOrUnion(CTypesData): + __slots__ = ['_blob'] + + @classmethod + def _create_ctype_obj(cls, init): + # may be overridden + raise TypeError("cannot instantiate opaque type %s" % (cls,)) + + def _get_own_repr(self): + return self._addr_repr(ctypes.addressof(self._blob)) + + @classmethod + def _offsetof(cls, fieldname): + return getattr(cls._ctype, fieldname).offset + + def _convert_to_address(self, BClass): + if getattr(BClass, '_BItem', None) is self.__class__: + return ctypes.addressof(self._blob) + else: + return CTypesData._convert_to_address(self, BClass) + + @classmethod + def _from_ctypes(cls, ctypes_struct_or_union): + self = cls.__new__(cls) + self._blob = ctypes_struct_or_union + return self + + @classmethod + def _to_ctypes(cls, value): + return value._blob + + def __repr__(self, c_name=None): + return CTypesData.__repr__(self, c_name or self._get_c_name(' &')) + + +class CTypesBackend(object): + + PRIMITIVE_TYPES = { + 'char': ctypes.c_char, + 'short': ctypes.c_short, + 'int': ctypes.c_int, + 'long': ctypes.c_long, + 'long long': ctypes.c_longlong, + 'signed char': ctypes.c_byte, + 'unsigned char': ctypes.c_ubyte, + 'unsigned short': ctypes.c_ushort, + 'unsigned int': ctypes.c_uint, + 'unsigned long': ctypes.c_ulong, + 'unsigned long long': ctypes.c_ulonglong, + 'float': ctypes.c_float, + 'double': ctypes.c_double, + '_Bool': ctypes.c_bool, + } + + for _name in ['unsigned long long', 'unsigned long', + 'unsigned int', 'unsigned short', 'unsigned char']: + _size = ctypes.sizeof(PRIMITIVE_TYPES[_name]) + PRIMITIVE_TYPES['uint%d_t' % (8*_size)] = PRIMITIVE_TYPES[_name] + if _size == ctypes.sizeof(ctypes.c_void_p): + PRIMITIVE_TYPES['uintptr_t'] = PRIMITIVE_TYPES[_name] + if _size == ctypes.sizeof(ctypes.c_size_t): + PRIMITIVE_TYPES['size_t'] = PRIMITIVE_TYPES[_name] + + for _name in ['long long', 'long', 'int', 'short', 'signed char']: + _size = ctypes.sizeof(PRIMITIVE_TYPES[_name]) + PRIMITIVE_TYPES['int%d_t' % (8*_size)] = PRIMITIVE_TYPES[_name] + if _size == ctypes.sizeof(ctypes.c_void_p): + PRIMITIVE_TYPES['intptr_t'] = PRIMITIVE_TYPES[_name] + PRIMITIVE_TYPES['ptrdiff_t'] = PRIMITIVE_TYPES[_name] + if _size == ctypes.sizeof(ctypes.c_size_t): + PRIMITIVE_TYPES['ssize_t'] = PRIMITIVE_TYPES[_name] + + + def __init__(self): + self.RTLD_LAZY = 0 # not supported anyway by ctypes + self.RTLD_NOW = 0 + self.RTLD_GLOBAL = ctypes.RTLD_GLOBAL + self.RTLD_LOCAL = ctypes.RTLD_LOCAL + + def set_ffi(self, ffi): + self.ffi = ffi + + def _get_types(self): + return CTypesData, CTypesType + + def load_library(self, path, flags=0): + cdll = ctypes.CDLL(path, flags) + return CTypesLibrary(self, cdll) + + def new_void_type(self): + class CTypesVoid(CTypesData): + __slots__ = [] + _reftypename = 'void &' + @staticmethod + def _from_ctypes(novalue): + return None + @staticmethod + def _to_ctypes(novalue): + if novalue is not None: + raise TypeError("None expected, got %s object" % + (type(novalue).__name__,)) + return None + CTypesVoid._fix_class() + return CTypesVoid + + def new_primitive_type(self, name): + if name == 'wchar_t': + raise NotImplementedError(name) + ctype = self.PRIMITIVE_TYPES[name] + if name == 'char': + kind = 'char' + elif name in ('float', 'double'): + kind = 'float' + else: + if name in ('signed char', 'unsigned char'): + kind = 'byte' + elif name == '_Bool': + kind = 'bool' + else: + kind = 'int' + is_signed = (ctype(-1).value == -1) + # + def _cast_source_to_int(source): + if isinstance(source, (int, long, float)): + source = int(source) + elif isinstance(source, CTypesData): + source = source._cast_to_integer() + elif isinstance(source, bytes): + source = ord(source) + elif source is None: + source = 0 + else: + raise TypeError("bad type for cast to %r: %r" % + (CTypesPrimitive, type(source).__name__)) + return source + # + kind1 = kind + class CTypesPrimitive(CTypesGenericPrimitive): + __slots__ = ['_value'] + _ctype = ctype + _reftypename = '%s &' % name + kind = kind1 + + def __init__(self, value): + self._value = value + + @staticmethod + def _create_ctype_obj(init): + if init is None: + return ctype() + return ctype(CTypesPrimitive._to_ctypes(init)) + + if kind == 'int' or kind == 'byte': + @classmethod + def _cast_from(cls, source): + source = _cast_source_to_int(source) + source = ctype(source).value # cast within range + return cls(source) + def __int__(self): + return self._value + + if kind == 'bool': + @classmethod + def _cast_from(cls, source): + if not isinstance(source, (int, long, float)): + source = _cast_source_to_int(source) + return cls(bool(source)) + def __int__(self): + return int(self._value) + + if kind == 'char': + @classmethod + def _cast_from(cls, source): + source = _cast_source_to_int(source) + source = bytechr(source & 0xFF) + return cls(source) + def __int__(self): + return ord(self._value) + + if kind == 'float': + @classmethod + def _cast_from(cls, source): + if isinstance(source, float): + pass + elif isinstance(source, CTypesGenericPrimitive): + if hasattr(source, '__float__'): + source = float(source) + else: + source = int(source) + else: + source = _cast_source_to_int(source) + source = ctype(source).value # fix precision + return cls(source) + def __int__(self): + return int(self._value) + def __float__(self): + return self._value + + _cast_to_integer = __int__ + + if kind == 'int' or kind == 'byte' or kind == 'bool': + @staticmethod + def _to_ctypes(x): + if not isinstance(x, (int, long)): + if isinstance(x, CTypesData): + x = int(x) + else: + raise TypeError("integer expected, got %s" % + type(x).__name__) + if ctype(x).value != x: + if not is_signed and x < 0: + raise OverflowError("%s: negative integer" % name) + else: + raise OverflowError("%s: integer out of bounds" + % name) + return x + + if kind == 'char': + @staticmethod + def _to_ctypes(x): + if isinstance(x, bytes) and len(x) == 1: + return x + if isinstance(x, CTypesPrimitive): # > + return x._value + raise TypeError("character expected, got %s" % + type(x).__name__) + def __nonzero__(self): + return ord(self._value) != 0 + else: + def __nonzero__(self): + return self._value != 0 + __bool__ = __nonzero__ + + if kind == 'float': + @staticmethod + def _to_ctypes(x): + if not isinstance(x, (int, long, float, CTypesData)): + raise TypeError("float expected, got %s" % + type(x).__name__) + return ctype(x).value + + @staticmethod + def _from_ctypes(value): + return getattr(value, 'value', value) + + @staticmethod + def _initialize(blob, init): + blob.value = CTypesPrimitive._to_ctypes(init) + + if kind == 'char': + def _to_string(self, maxlen): + return self._value + if kind == 'byte': + def _to_string(self, maxlen): + return chr(self._value & 0xff) + # + CTypesPrimitive._fix_class() + return CTypesPrimitive + + def new_pointer_type(self, BItem): + getbtype = self.ffi._get_cached_btype + if BItem is getbtype(model.PrimitiveType('char')): + kind = 'charp' + elif BItem in (getbtype(model.PrimitiveType('signed char')), + getbtype(model.PrimitiveType('unsigned char'))): + kind = 'bytep' + elif BItem is getbtype(model.void_type): + kind = 'voidp' + else: + kind = 'generic' + # + class CTypesPtr(CTypesGenericPtr): + __slots__ = ['_own'] + if kind == 'charp': + __slots__ += ['__as_strbuf'] + _BItem = BItem + if hasattr(BItem, '_ctype'): + _ctype = ctypes.POINTER(BItem._ctype) + _bitem_size = ctypes.sizeof(BItem._ctype) + else: + _ctype = ctypes.c_void_p + if issubclass(BItem, CTypesGenericArray): + _reftypename = BItem._get_c_name('(* &)') + else: + _reftypename = BItem._get_c_name(' * &') + + def __init__(self, init): + ctypeobj = BItem._create_ctype_obj(init) + if kind == 'charp': + self.__as_strbuf = ctypes.create_string_buffer( + ctypeobj.value + b'\x00') + self._as_ctype_ptr = ctypes.cast( + self.__as_strbuf, self._ctype) + else: + self._as_ctype_ptr = ctypes.pointer(ctypeobj) + self._address = ctypes.cast(self._as_ctype_ptr, + ctypes.c_void_p).value + self._own = True + + def __add__(self, other): + if isinstance(other, (int, long)): + return self._new_pointer_at(self._address + + other * self._bitem_size) + else: + return NotImplemented + + def __sub__(self, other): + if isinstance(other, (int, long)): + return self._new_pointer_at(self._address - + other * self._bitem_size) + elif type(self) is type(other): + return (self._address - other._address) // self._bitem_size + else: + return NotImplemented + + def __getitem__(self, index): + if getattr(self, '_own', False) and index != 0: + raise IndexError + return BItem._from_ctypes(self._as_ctype_ptr[index]) + + def __setitem__(self, index, value): + self._as_ctype_ptr[index] = BItem._to_ctypes(value) + + if kind == 'charp' or kind == 'voidp': + @classmethod + def _arg_to_ctypes(cls, *value): + if value and isinstance(value[0], bytes): + return ctypes.c_char_p(value[0]) + else: + return super(CTypesPtr, cls)._arg_to_ctypes(*value) + + if kind == 'charp' or kind == 'bytep': + def _to_string(self, maxlen): + if maxlen < 0: + maxlen = sys.maxsize + p = ctypes.cast(self._as_ctype_ptr, + ctypes.POINTER(ctypes.c_char)) + n = 0 + while n < maxlen and p[n] != b'\x00': + n += 1 + return b''.join([p[i] for i in range(n)]) + + def _get_own_repr(self): + if getattr(self, '_own', False): + return 'owning %d bytes' % ( + ctypes.sizeof(self._as_ctype_ptr.contents),) + return super(CTypesPtr, self)._get_own_repr() + # + if (BItem is self.ffi._get_cached_btype(model.void_type) or + BItem is self.ffi._get_cached_btype(model.PrimitiveType('char'))): + CTypesPtr._automatic_casts = True + # + CTypesPtr._fix_class() + return CTypesPtr + + def new_array_type(self, CTypesPtr, length): + if length is None: + brackets = ' &[]' + else: + brackets = ' &[%d]' % length + BItem = CTypesPtr._BItem + getbtype = self.ffi._get_cached_btype + if BItem is getbtype(model.PrimitiveType('char')): + kind = 'char' + elif BItem in (getbtype(model.PrimitiveType('signed char')), + getbtype(model.PrimitiveType('unsigned char'))): + kind = 'byte' + else: + kind = 'generic' + # + class CTypesArray(CTypesGenericArray): + __slots__ = ['_blob', '_own'] + if length is not None: + _ctype = BItem._ctype * length + else: + __slots__.append('_ctype') + _reftypename = BItem._get_c_name(brackets) + _declared_length = length + _CTPtr = CTypesPtr + + def __init__(self, init): + if length is None: + if isinstance(init, (int, long)): + len1 = init + init = None + elif kind == 'char' and isinstance(init, bytes): + len1 = len(init) + 1 # extra null + else: + init = tuple(init) + len1 = len(init) + self._ctype = BItem._ctype * len1 + self._blob = self._ctype() + self._own = True + if init is not None: + self._initialize(self._blob, init) + + @staticmethod + def _initialize(blob, init): + if isinstance(init, bytes): + init = [init[i:i+1] for i in range(len(init))] + else: + if isinstance(init, CTypesGenericArray): + if (len(init) != len(blob) or + not isinstance(init, CTypesArray)): + raise TypeError("length/type mismatch: %s" % (init,)) + init = tuple(init) + if len(init) > len(blob): + raise IndexError("too many initializers") + addr = ctypes.cast(blob, ctypes.c_void_p).value + PTR = ctypes.POINTER(BItem._ctype) + itemsize = ctypes.sizeof(BItem._ctype) + for i, value in enumerate(init): + p = ctypes.cast(addr + i * itemsize, PTR) + BItem._initialize(p.contents, value) + + def __len__(self): + return len(self._blob) + + def __getitem__(self, index): + if not (0 <= index < len(self._blob)): + raise IndexError + return BItem._from_ctypes(self._blob[index]) + + def __setitem__(self, index, value): + if not (0 <= index < len(self._blob)): + raise IndexError + self._blob[index] = BItem._to_ctypes(value) + + if kind == 'char' or kind == 'byte': + def _to_string(self, maxlen): + if maxlen < 0: + maxlen = len(self._blob) + p = ctypes.cast(self._blob, + ctypes.POINTER(ctypes.c_char)) + n = 0 + while n < maxlen and p[n] != b'\x00': + n += 1 + return b''.join([p[i] for i in range(n)]) + + def _get_own_repr(self): + if getattr(self, '_own', False): + return 'owning %d bytes' % (ctypes.sizeof(self._blob),) + return super(CTypesArray, self)._get_own_repr() + + def _convert_to_address(self, BClass): + if BClass in (CTypesPtr, None) or BClass._automatic_casts: + return ctypes.addressof(self._blob) + else: + return CTypesData._convert_to_address(self, BClass) + + @staticmethod + def _from_ctypes(ctypes_array): + self = CTypesArray.__new__(CTypesArray) + self._blob = ctypes_array + return self + + @staticmethod + def _arg_to_ctypes(value): + return CTypesPtr._arg_to_ctypes(value) + + def __add__(self, other): + if isinstance(other, (int, long)): + return CTypesPtr._new_pointer_at( + ctypes.addressof(self._blob) + + other * ctypes.sizeof(BItem._ctype)) + else: + return NotImplemented + + @classmethod + def _cast_from(cls, source): + raise NotImplementedError("casting to %r" % ( + cls._get_c_name(),)) + # + CTypesArray._fix_class() + return CTypesArray + + def _new_struct_or_union(self, kind, name, base_ctypes_class): + # + class struct_or_union(base_ctypes_class): + pass + struct_or_union.__name__ = '%s_%s' % (kind, name) + kind1 = kind + # + class CTypesStructOrUnion(CTypesBaseStructOrUnion): + __slots__ = ['_blob'] + _ctype = struct_or_union + _reftypename = '%s &' % (name,) + _kind = kind = kind1 + # + CTypesStructOrUnion._fix_class() + return CTypesStructOrUnion + + def new_struct_type(self, name): + return self._new_struct_or_union('struct', name, ctypes.Structure) + + def new_union_type(self, name): + return self._new_struct_or_union('union', name, ctypes.Union) + + def complete_struct_or_union(self, CTypesStructOrUnion, fields, tp, + totalsize=-1, totalalignment=-1, sflags=0, + pack=0): + if totalsize >= 0 or totalalignment >= 0: + raise NotImplementedError("the ctypes backend of CFFI does not support " + "structures completed by verify(); please " + "compile and install the _cffi_backend module.") + struct_or_union = CTypesStructOrUnion._ctype + fnames = [fname for (fname, BField, bitsize) in fields] + btypes = [BField for (fname, BField, bitsize) in fields] + bitfields = [bitsize for (fname, BField, bitsize) in fields] + # + bfield_types = {} + cfields = [] + for (fname, BField, bitsize) in fields: + if bitsize < 0: + cfields.append((fname, BField._ctype)) + bfield_types[fname] = BField + else: + cfields.append((fname, BField._ctype, bitsize)) + bfield_types[fname] = Ellipsis + if sflags & 8: + struct_or_union._pack_ = 1 + elif pack: + struct_or_union._pack_ = pack + struct_or_union._fields_ = cfields + CTypesStructOrUnion._bfield_types = bfield_types + # + @staticmethod + def _create_ctype_obj(init): + result = struct_or_union() + if init is not None: + initialize(result, init) + return result + CTypesStructOrUnion._create_ctype_obj = _create_ctype_obj + # + def initialize(blob, init): + if is_union: + if len(init) > 1: + raise ValueError("union initializer: %d items given, but " + "only one supported (use a dict if needed)" + % (len(init),)) + if not isinstance(init, dict): + if isinstance(init, (bytes, unicode)): + raise TypeError("union initializer: got a str") + init = tuple(init) + if len(init) > len(fnames): + raise ValueError("too many values for %s initializer" % + CTypesStructOrUnion._get_c_name()) + init = dict(zip(fnames, init)) + addr = ctypes.addressof(blob) + for fname, value in init.items(): + BField, bitsize = name2fieldtype[fname] + assert bitsize < 0, \ + "not implemented: initializer with bit fields" + offset = CTypesStructOrUnion._offsetof(fname) + PTR = ctypes.POINTER(BField._ctype) + p = ctypes.cast(addr + offset, PTR) + BField._initialize(p.contents, value) + is_union = CTypesStructOrUnion._kind == 'union' + name2fieldtype = dict(zip(fnames, zip(btypes, bitfields))) + # + for fname, BField, bitsize in fields: + if fname == '': + raise NotImplementedError("nested anonymous structs/unions") + if hasattr(CTypesStructOrUnion, fname): + raise ValueError("the field name %r conflicts in " + "the ctypes backend" % fname) + if bitsize < 0: + def getter(self, fname=fname, BField=BField, + offset=CTypesStructOrUnion._offsetof(fname), + PTR=ctypes.POINTER(BField._ctype)): + addr = ctypes.addressof(self._blob) + p = ctypes.cast(addr + offset, PTR) + return BField._from_ctypes(p.contents) + def setter(self, value, fname=fname, BField=BField): + setattr(self._blob, fname, BField._to_ctypes(value)) + # + if issubclass(BField, CTypesGenericArray): + setter = None + if BField._declared_length == 0: + def getter(self, fname=fname, BFieldPtr=BField._CTPtr, + offset=CTypesStructOrUnion._offsetof(fname), + PTR=ctypes.POINTER(BField._ctype)): + addr = ctypes.addressof(self._blob) + p = ctypes.cast(addr + offset, PTR) + return BFieldPtr._from_ctypes(p) + # + else: + def getter(self, fname=fname, BField=BField): + return BField._from_ctypes(getattr(self._blob, fname)) + def setter(self, value, fname=fname, BField=BField): + # xxx obscure workaround + value = BField._to_ctypes(value) + oldvalue = getattr(self._blob, fname) + setattr(self._blob, fname, value) + if value != getattr(self._blob, fname): + setattr(self._blob, fname, oldvalue) + raise OverflowError("value too large for bitfield") + setattr(CTypesStructOrUnion, fname, property(getter, setter)) + # + CTypesPtr = self.ffi._get_cached_btype(model.PointerType(tp)) + for fname in fnames: + if hasattr(CTypesPtr, fname): + raise ValueError("the field name %r conflicts in " + "the ctypes backend" % fname) + def getter(self, fname=fname): + return getattr(self[0], fname) + def setter(self, value, fname=fname): + setattr(self[0], fname, value) + setattr(CTypesPtr, fname, property(getter, setter)) + + def new_function_type(self, BArgs, BResult, has_varargs): + nameargs = [BArg._get_c_name() for BArg in BArgs] + if has_varargs: + nameargs.append('...') + nameargs = ', '.join(nameargs) + # + class CTypesFunctionPtr(CTypesGenericPtr): + __slots__ = ['_own_callback', '_name'] + _ctype = ctypes.CFUNCTYPE(getattr(BResult, '_ctype', None), + *[BArg._ctype for BArg in BArgs], + use_errno=True) + _reftypename = BResult._get_c_name('(* &)(%s)' % (nameargs,)) + + def __init__(self, init, error=None): + # create a callback to the Python callable init() + import traceback + assert not has_varargs, "varargs not supported for callbacks" + if getattr(BResult, '_ctype', None) is not None: + error = BResult._from_ctypes( + BResult._create_ctype_obj(error)) + else: + error = None + def callback(*args): + args2 = [] + for arg, BArg in zip(args, BArgs): + args2.append(BArg._from_ctypes(arg)) + try: + res2 = init(*args2) + res2 = BResult._to_ctypes(res2) + except: + traceback.print_exc() + res2 = error + if issubclass(BResult, CTypesGenericPtr): + if res2: + res2 = ctypes.cast(res2, ctypes.c_void_p).value + # .value: http://bugs.python.org/issue1574593 + else: + res2 = None + #print repr(res2) + return res2 + if issubclass(BResult, CTypesGenericPtr): + # The only pointers callbacks can return are void*s: + # http://bugs.python.org/issue5710 + callback_ctype = ctypes.CFUNCTYPE( + ctypes.c_void_p, + *[BArg._ctype for BArg in BArgs], + use_errno=True) + else: + callback_ctype = CTypesFunctionPtr._ctype + self._as_ctype_ptr = callback_ctype(callback) + self._address = ctypes.cast(self._as_ctype_ptr, + ctypes.c_void_p).value + self._own_callback = init + + @staticmethod + def _initialize(ctypes_ptr, value): + if value: + raise NotImplementedError("ctypes backend: not supported: " + "initializers for function pointers") + + def __repr__(self): + c_name = getattr(self, '_name', None) + if c_name: + i = self._reftypename.index('(* &)') + if self._reftypename[i-1] not in ' )*': + c_name = ' ' + c_name + c_name = self._reftypename.replace('(* &)', c_name) + return CTypesData.__repr__(self, c_name) + + def _get_own_repr(self): + if getattr(self, '_own_callback', None) is not None: + return 'calling %r' % (self._own_callback,) + return super(CTypesFunctionPtr, self)._get_own_repr() + + def __call__(self, *args): + if has_varargs: + assert len(args) >= len(BArgs) + extraargs = args[len(BArgs):] + args = args[:len(BArgs)] + else: + assert len(args) == len(BArgs) + ctypes_args = [] + for arg, BArg in zip(args, BArgs): + ctypes_args.append(BArg._arg_to_ctypes(arg)) + if has_varargs: + for i, arg in enumerate(extraargs): + if arg is None: + ctypes_args.append(ctypes.c_void_p(0)) # NULL + continue + if not isinstance(arg, CTypesData): + raise TypeError( + "argument %d passed in the variadic part " + "needs to be a cdata object (got %s)" % + (1 + len(BArgs) + i, type(arg).__name__)) + ctypes_args.append(arg._arg_to_ctypes(arg)) + result = self._as_ctype_ptr(*ctypes_args) + return BResult._from_ctypes(result) + # + CTypesFunctionPtr._fix_class() + return CTypesFunctionPtr + + def new_enum_type(self, name, enumerators, enumvalues, CTypesInt): + assert isinstance(name, str) + reverse_mapping = dict(zip(reversed(enumvalues), + reversed(enumerators))) + # + class CTypesEnum(CTypesInt): + __slots__ = [] + _reftypename = '%s &' % name + + def _get_own_repr(self): + value = self._value + try: + return '%d: %s' % (value, reverse_mapping[value]) + except KeyError: + return str(value) + + def _to_string(self, maxlen): + value = self._value + try: + return reverse_mapping[value] + except KeyError: + return str(value) + # + CTypesEnum._fix_class() + return CTypesEnum + + def get_errno(self): + return ctypes.get_errno() + + def set_errno(self, value): + ctypes.set_errno(value) + + def string(self, b, maxlen=-1): + return b._to_string(maxlen) + + def buffer(self, bptr, size=-1): + raise NotImplementedError("buffer() with ctypes backend") + + def sizeof(self, cdata_or_BType): + if isinstance(cdata_or_BType, CTypesData): + return cdata_or_BType._get_size_of_instance() + else: + assert issubclass(cdata_or_BType, CTypesData) + return cdata_or_BType._get_size() + + def alignof(self, BType): + assert issubclass(BType, CTypesData) + return BType._alignment() + + def newp(self, BType, source): + if not issubclass(BType, CTypesData): + raise TypeError + return BType._newp(source) + + def cast(self, BType, source): + return BType._cast_from(source) + + def callback(self, BType, source, error, onerror): + assert onerror is None # XXX not implemented + return BType(source, error) + + _weakref_cache_ref = None + + def gcp(self, cdata, destructor, size=0): + if self._weakref_cache_ref is None: + import weakref + class MyRef(weakref.ref): + def __eq__(self, other): + myref = self() + return self is other or ( + myref is not None and myref is other()) + def __ne__(self, other): + return not (self == other) + def __hash__(self): + try: + return self._hash + except AttributeError: + self._hash = hash(self()) + return self._hash + self._weakref_cache_ref = {}, MyRef + weak_cache, MyRef = self._weakref_cache_ref + + if destructor is None: + try: + del weak_cache[MyRef(cdata)] + except KeyError: + raise TypeError("Can remove destructor only on a object " + "previously returned by ffi.gc()") + return None + + def remove(k): + cdata, destructor = weak_cache.pop(k, (None, None)) + if destructor is not None: + destructor(cdata) + + new_cdata = self.cast(self.typeof(cdata), cdata) + assert new_cdata is not cdata + weak_cache[MyRef(new_cdata, remove)] = (cdata, destructor) + return new_cdata + + typeof = type + + def getcname(self, BType, replace_with): + return BType._get_c_name(replace_with) + + def typeoffsetof(self, BType, fieldname, num=0): + if isinstance(fieldname, str): + if num == 0 and issubclass(BType, CTypesGenericPtr): + BType = BType._BItem + if not issubclass(BType, CTypesBaseStructOrUnion): + raise TypeError("expected a struct or union ctype") + BField = BType._bfield_types[fieldname] + if BField is Ellipsis: + raise TypeError("not supported for bitfields") + return (BField, BType._offsetof(fieldname)) + elif isinstance(fieldname, (int, long)): + if issubclass(BType, CTypesGenericArray): + BType = BType._CTPtr + if not issubclass(BType, CTypesGenericPtr): + raise TypeError("expected an array or ptr ctype") + BItem = BType._BItem + offset = BItem._get_size() * fieldname + if offset > sys.maxsize: + raise OverflowError + return (BItem, offset) + else: + raise TypeError(type(fieldname)) + + def rawaddressof(self, BTypePtr, cdata, offset=None): + if isinstance(cdata, CTypesBaseStructOrUnion): + ptr = ctypes.pointer(type(cdata)._to_ctypes(cdata)) + elif isinstance(cdata, CTypesGenericPtr): + if offset is None or not issubclass(type(cdata)._BItem, + CTypesBaseStructOrUnion): + raise TypeError("unexpected cdata type") + ptr = type(cdata)._to_ctypes(cdata) + elif isinstance(cdata, CTypesGenericArray): + ptr = type(cdata)._to_ctypes(cdata) + else: + raise TypeError("expected a ") + if offset: + ptr = ctypes.cast( + ctypes.c_void_p( + ctypes.cast(ptr, ctypes.c_void_p).value + offset), + type(ptr)) + return BTypePtr._from_ctypes(ptr) + + +class CTypesLibrary(object): + + def __init__(self, backend, cdll): + self.backend = backend + self.cdll = cdll + + def load_function(self, BType, name): + c_func = getattr(self.cdll, name) + funcobj = BType._from_ctypes(c_func) + funcobj._name = name + return funcobj + + def read_variable(self, BType, name): + try: + ctypes_obj = BType._ctype.in_dll(self.cdll, name) + except AttributeError as e: + raise NotImplementedError(e) + return BType._from_ctypes(ctypes_obj) + + def write_variable(self, BType, name, value): + new_ctypes_obj = BType._to_ctypes(value) + ctypes_obj = BType._ctype.in_dll(self.cdll, name) + ctypes.memmove(ctypes.addressof(ctypes_obj), + ctypes.addressof(new_ctypes_obj), + ctypes.sizeof(BType._ctype)) diff --git a/WebCrawler/venv/Lib/site-packages/cffi/cffi_opcode.py b/WebCrawler/venv/Lib/site-packages/cffi/cffi_opcode.py new file mode 100644 index 0000000..a0df98d --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/cffi/cffi_opcode.py @@ -0,0 +1,187 @@ +from .error import VerificationError + +class CffiOp(object): + def __init__(self, op, arg): + self.op = op + self.arg = arg + + def as_c_expr(self): + if self.op is None: + assert isinstance(self.arg, str) + return '(_cffi_opcode_t)(%s)' % (self.arg,) + classname = CLASS_NAME[self.op] + return '_CFFI_OP(_CFFI_OP_%s, %s)' % (classname, self.arg) + + def as_python_bytes(self): + if self.op is None and self.arg.isdigit(): + value = int(self.arg) # non-negative: '-' not in self.arg + if value >= 2**31: + raise OverflowError("cannot emit %r: limited to 2**31-1" + % (self.arg,)) + return format_four_bytes(value) + if isinstance(self.arg, str): + raise VerificationError("cannot emit to Python: %r" % (self.arg,)) + return format_four_bytes((self.arg << 8) | self.op) + + def __str__(self): + classname = CLASS_NAME.get(self.op, self.op) + return '(%s %s)' % (classname, self.arg) + +def format_four_bytes(num): + return '\\x%02X\\x%02X\\x%02X\\x%02X' % ( + (num >> 24) & 0xFF, + (num >> 16) & 0xFF, + (num >> 8) & 0xFF, + (num ) & 0xFF) + +OP_PRIMITIVE = 1 +OP_POINTER = 3 +OP_ARRAY = 5 +OP_OPEN_ARRAY = 7 +OP_STRUCT_UNION = 9 +OP_ENUM = 11 +OP_FUNCTION = 13 +OP_FUNCTION_END = 15 +OP_NOOP = 17 +OP_BITFIELD = 19 +OP_TYPENAME = 21 +OP_CPYTHON_BLTN_V = 23 # varargs +OP_CPYTHON_BLTN_N = 25 # noargs +OP_CPYTHON_BLTN_O = 27 # O (i.e. a single arg) +OP_CONSTANT = 29 +OP_CONSTANT_INT = 31 +OP_GLOBAL_VAR = 33 +OP_DLOPEN_FUNC = 35 +OP_DLOPEN_CONST = 37 +OP_GLOBAL_VAR_F = 39 +OP_EXTERN_PYTHON = 41 + +PRIM_VOID = 0 +PRIM_BOOL = 1 +PRIM_CHAR = 2 +PRIM_SCHAR = 3 +PRIM_UCHAR = 4 +PRIM_SHORT = 5 +PRIM_USHORT = 6 +PRIM_INT = 7 +PRIM_UINT = 8 +PRIM_LONG = 9 +PRIM_ULONG = 10 +PRIM_LONGLONG = 11 +PRIM_ULONGLONG = 12 +PRIM_FLOAT = 13 +PRIM_DOUBLE = 14 +PRIM_LONGDOUBLE = 15 + +PRIM_WCHAR = 16 +PRIM_INT8 = 17 +PRIM_UINT8 = 18 +PRIM_INT16 = 19 +PRIM_UINT16 = 20 +PRIM_INT32 = 21 +PRIM_UINT32 = 22 +PRIM_INT64 = 23 +PRIM_UINT64 = 24 +PRIM_INTPTR = 25 +PRIM_UINTPTR = 26 +PRIM_PTRDIFF = 27 +PRIM_SIZE = 28 +PRIM_SSIZE = 29 +PRIM_INT_LEAST8 = 30 +PRIM_UINT_LEAST8 = 31 +PRIM_INT_LEAST16 = 32 +PRIM_UINT_LEAST16 = 33 +PRIM_INT_LEAST32 = 34 +PRIM_UINT_LEAST32 = 35 +PRIM_INT_LEAST64 = 36 +PRIM_UINT_LEAST64 = 37 +PRIM_INT_FAST8 = 38 +PRIM_UINT_FAST8 = 39 +PRIM_INT_FAST16 = 40 +PRIM_UINT_FAST16 = 41 +PRIM_INT_FAST32 = 42 +PRIM_UINT_FAST32 = 43 +PRIM_INT_FAST64 = 44 +PRIM_UINT_FAST64 = 45 +PRIM_INTMAX = 46 +PRIM_UINTMAX = 47 +PRIM_FLOATCOMPLEX = 48 +PRIM_DOUBLECOMPLEX = 49 +PRIM_CHAR16 = 50 +PRIM_CHAR32 = 51 + +_NUM_PRIM = 52 +_UNKNOWN_PRIM = -1 +_UNKNOWN_FLOAT_PRIM = -2 +_UNKNOWN_LONG_DOUBLE = -3 + +_IO_FILE_STRUCT = -1 + +PRIMITIVE_TO_INDEX = { + 'char': PRIM_CHAR, + 'short': PRIM_SHORT, + 'int': PRIM_INT, + 'long': PRIM_LONG, + 'long long': PRIM_LONGLONG, + 'signed char': PRIM_SCHAR, + 'unsigned char': PRIM_UCHAR, + 'unsigned short': PRIM_USHORT, + 'unsigned int': PRIM_UINT, + 'unsigned long': PRIM_ULONG, + 'unsigned long long': PRIM_ULONGLONG, + 'float': PRIM_FLOAT, + 'double': PRIM_DOUBLE, + 'long double': PRIM_LONGDOUBLE, + 'float _Complex': PRIM_FLOATCOMPLEX, + 'double _Complex': PRIM_DOUBLECOMPLEX, + '_Bool': PRIM_BOOL, + 'wchar_t': PRIM_WCHAR, + 'char16_t': PRIM_CHAR16, + 'char32_t': PRIM_CHAR32, + 'int8_t': PRIM_INT8, + 'uint8_t': PRIM_UINT8, + 'int16_t': PRIM_INT16, + 'uint16_t': PRIM_UINT16, + 'int32_t': PRIM_INT32, + 'uint32_t': PRIM_UINT32, + 'int64_t': PRIM_INT64, + 'uint64_t': PRIM_UINT64, + 'intptr_t': PRIM_INTPTR, + 'uintptr_t': PRIM_UINTPTR, + 'ptrdiff_t': PRIM_PTRDIFF, + 'size_t': PRIM_SIZE, + 'ssize_t': PRIM_SSIZE, + 'int_least8_t': PRIM_INT_LEAST8, + 'uint_least8_t': PRIM_UINT_LEAST8, + 'int_least16_t': PRIM_INT_LEAST16, + 'uint_least16_t': PRIM_UINT_LEAST16, + 'int_least32_t': PRIM_INT_LEAST32, + 'uint_least32_t': PRIM_UINT_LEAST32, + 'int_least64_t': PRIM_INT_LEAST64, + 'uint_least64_t': PRIM_UINT_LEAST64, + 'int_fast8_t': PRIM_INT_FAST8, + 'uint_fast8_t': PRIM_UINT_FAST8, + 'int_fast16_t': PRIM_INT_FAST16, + 'uint_fast16_t': PRIM_UINT_FAST16, + 'int_fast32_t': PRIM_INT_FAST32, + 'uint_fast32_t': PRIM_UINT_FAST32, + 'int_fast64_t': PRIM_INT_FAST64, + 'uint_fast64_t': PRIM_UINT_FAST64, + 'intmax_t': PRIM_INTMAX, + 'uintmax_t': PRIM_UINTMAX, + } + +F_UNION = 0x01 +F_CHECK_FIELDS = 0x02 +F_PACKED = 0x04 +F_EXTERNAL = 0x08 +F_OPAQUE = 0x10 + +G_FLAGS = dict([('_CFFI_' + _key, globals()[_key]) + for _key in ['F_UNION', 'F_CHECK_FIELDS', 'F_PACKED', + 'F_EXTERNAL', 'F_OPAQUE']]) + +CLASS_NAME = {} +for _name, _value in list(globals().items()): + if _name.startswith('OP_') and isinstance(_value, int): + CLASS_NAME[_value] = _name[3:] diff --git a/WebCrawler/venv/Lib/site-packages/cffi/commontypes.py b/WebCrawler/venv/Lib/site-packages/cffi/commontypes.py new file mode 100644 index 0000000..8ec97c7 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/cffi/commontypes.py @@ -0,0 +1,80 @@ +import sys +from . import model +from .error import FFIError + + +COMMON_TYPES = {} + +try: + # fetch "bool" and all simple Windows types + from _cffi_backend import _get_common_types + _get_common_types(COMMON_TYPES) +except ImportError: + pass + +COMMON_TYPES['FILE'] = model.unknown_type('FILE', '_IO_FILE') +COMMON_TYPES['bool'] = '_Bool' # in case we got ImportError above + +for _type in model.PrimitiveType.ALL_PRIMITIVE_TYPES: + if _type.endswith('_t'): + COMMON_TYPES[_type] = _type +del _type + +_CACHE = {} + +def resolve_common_type(parser, commontype): + try: + return _CACHE[commontype] + except KeyError: + cdecl = COMMON_TYPES.get(commontype, commontype) + if not isinstance(cdecl, str): + result, quals = cdecl, 0 # cdecl is already a BaseType + elif cdecl in model.PrimitiveType.ALL_PRIMITIVE_TYPES: + result, quals = model.PrimitiveType(cdecl), 0 + elif cdecl == 'set-unicode-needed': + raise FFIError("The Windows type %r is only available after " + "you call ffi.set_unicode()" % (commontype,)) + else: + if commontype == cdecl: + raise FFIError( + "Unsupported type: %r. Please look at " + "http://cffi.readthedocs.io/en/latest/cdef.html#ffi-cdef-limitations " + "and file an issue if you think this type should really " + "be supported." % (commontype,)) + result, quals = parser.parse_type_and_quals(cdecl) # recursive + + assert isinstance(result, model.BaseTypeByIdentity) + _CACHE[commontype] = result, quals + return result, quals + + +# ____________________________________________________________ +# extra types for Windows (most of them are in commontypes.c) + + +def win_common_types(): + return { + "UNICODE_STRING": model.StructType( + "_UNICODE_STRING", + ["Length", + "MaximumLength", + "Buffer"], + [model.PrimitiveType("unsigned short"), + model.PrimitiveType("unsigned short"), + model.PointerType(model.PrimitiveType("wchar_t"))], + [-1, -1, -1]), + "PUNICODE_STRING": "UNICODE_STRING *", + "PCUNICODE_STRING": "const UNICODE_STRING *", + + "TBYTE": "set-unicode-needed", + "TCHAR": "set-unicode-needed", + "LPCTSTR": "set-unicode-needed", + "PCTSTR": "set-unicode-needed", + "LPTSTR": "set-unicode-needed", + "PTSTR": "set-unicode-needed", + "PTBYTE": "set-unicode-needed", + "PTCHAR": "set-unicode-needed", + } + +if sys.platform == 'win32': + COMMON_TYPES.update(win_common_types()) diff --git a/WebCrawler/venv/Lib/site-packages/cffi/cparser.py b/WebCrawler/venv/Lib/site-packages/cffi/cparser.py new file mode 100644 index 0000000..74830e9 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/cffi/cparser.py @@ -0,0 +1,1006 @@ +from . import model +from .commontypes import COMMON_TYPES, resolve_common_type +from .error import FFIError, CDefError +try: + from . import _pycparser as pycparser +except ImportError: + import pycparser +import weakref, re, sys + +try: + if sys.version_info < (3,): + import thread as _thread + else: + import _thread + lock = _thread.allocate_lock() +except ImportError: + lock = None + +def _workaround_for_static_import_finders(): + # Issue #392: packaging tools like cx_Freeze can not find these + # because pycparser uses exec dynamic import. This is an obscure + # workaround. This function is never called. + import pycparser.yacctab + import pycparser.lextab + +CDEF_SOURCE_STRING = "" +_r_comment = re.compile(r"/\*.*?\*/|//([^\n\\]|\\.)*?$", + re.DOTALL | re.MULTILINE) +_r_define = re.compile(r"^\s*#\s*define\s+([A-Za-z_][A-Za-z_0-9]*)" + r"\b((?:[^\n\\]|\\.)*?)$", + re.DOTALL | re.MULTILINE) +_r_line_directive = re.compile(r"^[ \t]*#[ \t]*(?:line|\d+)\b.*$", re.MULTILINE) +_r_partial_enum = re.compile(r"=\s*\.\.\.\s*[,}]|\.\.\.\s*\}") +_r_enum_dotdotdot = re.compile(r"__dotdotdot\d+__$") +_r_partial_array = re.compile(r"\[\s*\.\.\.\s*\]") +_r_words = re.compile(r"\w+|\S") +_parser_cache = None +_r_int_literal = re.compile(r"-?0?x?[0-9a-f]+[lu]*$", re.IGNORECASE) +_r_stdcall1 = re.compile(r"\b(__stdcall|WINAPI)\b") +_r_stdcall2 = re.compile(r"[(]\s*(__stdcall|WINAPI)\b") +_r_cdecl = re.compile(r"\b__cdecl\b") +_r_extern_python = re.compile(r'\bextern\s*"' + r'(Python|Python\s*\+\s*C|C\s*\+\s*Python)"\s*.') +_r_star_const_space = re.compile( # matches "* const " + r"[*]\s*((const|volatile|restrict)\b\s*)+") +_r_int_dotdotdot = re.compile(r"(\b(int|long|short|signed|unsigned|char)\s*)+" + r"\.\.\.") +_r_float_dotdotdot = re.compile(r"\b(double|float)\s*\.\.\.") + +def _get_parser(): + global _parser_cache + if _parser_cache is None: + _parser_cache = pycparser.CParser() + return _parser_cache + +def _workaround_for_old_pycparser(csource): + # Workaround for a pycparser issue (fixed between pycparser 2.10 and + # 2.14): "char*const***" gives us a wrong syntax tree, the same as + # for "char***(*const)". This means we can't tell the difference + # afterwards. But "char(*const(***))" gives us the right syntax + # tree. The issue only occurs if there are several stars in + # sequence with no parenthesis inbetween, just possibly qualifiers. + # Attempt to fix it by adding some parentheses in the source: each + # time we see "* const" or "* const *", we add an opening + # parenthesis before each star---the hard part is figuring out where + # to close them. + parts = [] + while True: + match = _r_star_const_space.search(csource) + if not match: + break + #print repr(''.join(parts)+csource), '=>', + parts.append(csource[:match.start()]) + parts.append('('); closing = ')' + parts.append(match.group()) # e.g. "* const " + endpos = match.end() + if csource.startswith('*', endpos): + parts.append('('); closing += ')' + level = 0 + i = endpos + while i < len(csource): + c = csource[i] + if c == '(': + level += 1 + elif c == ')': + if level == 0: + break + level -= 1 + elif c in ',;=': + if level == 0: + break + i += 1 + csource = csource[endpos:i] + closing + csource[i:] + #print repr(''.join(parts)+csource) + parts.append(csource) + return ''.join(parts) + +def _preprocess_extern_python(csource): + # input: `extern "Python" int foo(int);` or + # `extern "Python" { int foo(int); }` + # output: + # void __cffi_extern_python_start; + # int foo(int); + # void __cffi_extern_python_stop; + # + # input: `extern "Python+C" int foo(int);` + # output: + # void __cffi_extern_python_plus_c_start; + # int foo(int); + # void __cffi_extern_python_stop; + parts = [] + while True: + match = _r_extern_python.search(csource) + if not match: + break + endpos = match.end() - 1 + #print + #print ''.join(parts)+csource + #print '=>' + parts.append(csource[:match.start()]) + if 'C' in match.group(1): + parts.append('void __cffi_extern_python_plus_c_start; ') + else: + parts.append('void __cffi_extern_python_start; ') + if csource[endpos] == '{': + # grouping variant + closing = csource.find('}', endpos) + if closing < 0: + raise CDefError("'extern \"Python\" {': no '}' found") + if csource.find('{', endpos + 1, closing) >= 0: + raise NotImplementedError("cannot use { } inside a block " + "'extern \"Python\" { ... }'") + parts.append(csource[endpos+1:closing]) + csource = csource[closing+1:] + else: + # non-grouping variant + semicolon = csource.find(';', endpos) + if semicolon < 0: + raise CDefError("'extern \"Python\": no ';' found") + parts.append(csource[endpos:semicolon+1]) + csource = csource[semicolon+1:] + parts.append(' void __cffi_extern_python_stop;') + #print ''.join(parts)+csource + #print + parts.append(csource) + return ''.join(parts) + +def _warn_for_string_literal(csource): + if '"' not in csource: + return + for line in csource.splitlines(): + if '"' in line and not line.lstrip().startswith('#'): + import warnings + warnings.warn("String literal found in cdef() or type source. " + "String literals are ignored here, but you should " + "remove them anyway because some character sequences " + "confuse pre-parsing.") + break + +def _warn_for_non_extern_non_static_global_variable(decl): + if not decl.storage: + import warnings + warnings.warn("Global variable '%s' in cdef(): for consistency " + "with C it should have a storage class specifier " + "(usually 'extern')" % (decl.name,)) + +def _remove_line_directives(csource): + # _r_line_directive matches whole lines, without the final \n, if they + # start with '#line' with some spacing allowed, or '#NUMBER'. This + # function stores them away and replaces them with exactly the string + # '#line@N', where N is the index in the list 'line_directives'. + line_directives = [] + def replace(m): + i = len(line_directives) + line_directives.append(m.group()) + return '#line@%d' % i + csource = _r_line_directive.sub(replace, csource) + return csource, line_directives + +def _put_back_line_directives(csource, line_directives): + def replace(m): + s = m.group() + if not s.startswith('#line@'): + raise AssertionError("unexpected #line directive " + "(should have been processed and removed") + return line_directives[int(s[6:])] + return _r_line_directive.sub(replace, csource) + +def _preprocess(csource): + # First, remove the lines of the form '#line N "filename"' because + # the "filename" part could confuse the rest + csource, line_directives = _remove_line_directives(csource) + # Remove comments. NOTE: this only work because the cdef() section + # should not contain any string literals (except in line directives)! + def replace_keeping_newlines(m): + return ' ' + m.group().count('\n') * '\n' + csource = _r_comment.sub(replace_keeping_newlines, csource) + # Remove the "#define FOO x" lines + macros = {} + for match in _r_define.finditer(csource): + macroname, macrovalue = match.groups() + macrovalue = macrovalue.replace('\\\n', '').strip() + macros[macroname] = macrovalue + csource = _r_define.sub('', csource) + # + if pycparser.__version__ < '2.14': + csource = _workaround_for_old_pycparser(csource) + # + # BIG HACK: replace WINAPI or __stdcall with "volatile const". + # It doesn't make sense for the return type of a function to be + # "volatile volatile const", so we abuse it to detect __stdcall... + # Hack number 2 is that "int(volatile *fptr)();" is not valid C + # syntax, so we place the "volatile" before the opening parenthesis. + csource = _r_stdcall2.sub(' volatile volatile const(', csource) + csource = _r_stdcall1.sub(' volatile volatile const ', csource) + csource = _r_cdecl.sub(' ', csource) + # + # Replace `extern "Python"` with start/end markers + csource = _preprocess_extern_python(csource) + # + # Now there should not be any string literal left; warn if we get one + _warn_for_string_literal(csource) + # + # Replace "[...]" with "[__dotdotdotarray__]" + csource = _r_partial_array.sub('[__dotdotdotarray__]', csource) + # + # Replace "...}" with "__dotdotdotNUM__}". This construction should + # occur only at the end of enums; at the end of structs we have "...;}" + # and at the end of vararg functions "...);". Also replace "=...[,}]" + # with ",__dotdotdotNUM__[,}]": this occurs in the enums too, when + # giving an unknown value. + matches = list(_r_partial_enum.finditer(csource)) + for number, match in enumerate(reversed(matches)): + p = match.start() + if csource[p] == '=': + p2 = csource.find('...', p, match.end()) + assert p2 > p + csource = '%s,__dotdotdot%d__ %s' % (csource[:p], number, + csource[p2+3:]) + else: + assert csource[p:p+3] == '...' + csource = '%s __dotdotdot%d__ %s' % (csource[:p], number, + csource[p+3:]) + # Replace "int ..." or "unsigned long int..." with "__dotdotdotint__" + csource = _r_int_dotdotdot.sub(' __dotdotdotint__ ', csource) + # Replace "float ..." or "double..." with "__dotdotdotfloat__" + csource = _r_float_dotdotdot.sub(' __dotdotdotfloat__ ', csource) + # Replace all remaining "..." with the same name, "__dotdotdot__", + # which is declared with a typedef for the purpose of C parsing. + csource = csource.replace('...', ' __dotdotdot__ ') + # Finally, put back the line directives + csource = _put_back_line_directives(csource, line_directives) + return csource, macros + +def _common_type_names(csource): + # Look in the source for what looks like usages of types from the + # list of common types. A "usage" is approximated here as the + # appearance of the word, minus a "definition" of the type, which + # is the last word in a "typedef" statement. Approximative only + # but should be fine for all the common types. + look_for_words = set(COMMON_TYPES) + look_for_words.add(';') + look_for_words.add(',') + look_for_words.add('(') + look_for_words.add(')') + look_for_words.add('typedef') + words_used = set() + is_typedef = False + paren = 0 + previous_word = '' + for word in _r_words.findall(csource): + if word in look_for_words: + if word == ';': + if is_typedef: + words_used.discard(previous_word) + look_for_words.discard(previous_word) + is_typedef = False + elif word == 'typedef': + is_typedef = True + paren = 0 + elif word == '(': + paren += 1 + elif word == ')': + paren -= 1 + elif word == ',': + if is_typedef and paren == 0: + words_used.discard(previous_word) + look_for_words.discard(previous_word) + else: # word in COMMON_TYPES + words_used.add(word) + previous_word = word + return words_used + + +class Parser(object): + + def __init__(self): + self._declarations = {} + self._included_declarations = set() + self._anonymous_counter = 0 + self._structnode2type = weakref.WeakKeyDictionary() + self._options = {} + self._int_constants = {} + self._recomplete = [] + self._uses_new_feature = None + + def _parse(self, csource): + csource, macros = _preprocess(csource) + # XXX: for more efficiency we would need to poke into the + # internals of CParser... the following registers the + # typedefs, because their presence or absence influences the + # parsing itself (but what they are typedef'ed to plays no role) + ctn = _common_type_names(csource) + typenames = [] + for name in sorted(self._declarations): + if name.startswith('typedef '): + name = name[8:] + typenames.append(name) + ctn.discard(name) + typenames += sorted(ctn) + # + csourcelines = [] + csourcelines.append('# 1 ""') + for typename in typenames: + csourcelines.append('typedef int %s;' % typename) + csourcelines.append('typedef int __dotdotdotint__, __dotdotdotfloat__,' + ' __dotdotdot__;') + # this forces pycparser to consider the following in the file + # called from line 1 + csourcelines.append('# 1 "%s"' % (CDEF_SOURCE_STRING,)) + csourcelines.append(csource) + fullcsource = '\n'.join(csourcelines) + if lock is not None: + lock.acquire() # pycparser is not thread-safe... + try: + ast = _get_parser().parse(fullcsource) + except pycparser.c_parser.ParseError as e: + self.convert_pycparser_error(e, csource) + finally: + if lock is not None: + lock.release() + # csource will be used to find buggy source text + return ast, macros, csource + + def _convert_pycparser_error(self, e, csource): + # xxx look for ":NUM:" at the start of str(e) + # and interpret that as a line number. This will not work if + # the user gives explicit ``# NUM "FILE"`` directives. + line = None + msg = str(e) + match = re.match(r"%s:(\d+):" % (CDEF_SOURCE_STRING,), msg) + if match: + linenum = int(match.group(1), 10) + csourcelines = csource.splitlines() + if 1 <= linenum <= len(csourcelines): + line = csourcelines[linenum-1] + return line + + def convert_pycparser_error(self, e, csource): + line = self._convert_pycparser_error(e, csource) + + msg = str(e) + if line: + msg = 'cannot parse "%s"\n%s' % (line.strip(), msg) + else: + msg = 'parse error\n%s' % (msg,) + raise CDefError(msg) + + def parse(self, csource, override=False, packed=False, pack=None, + dllexport=False): + if packed: + if packed != True: + raise ValueError("'packed' should be False or True; use " + "'pack' to give another value") + if pack: + raise ValueError("cannot give both 'pack' and 'packed'") + pack = 1 + elif pack: + if pack & (pack - 1): + raise ValueError("'pack' must be a power of two, not %r" % + (pack,)) + else: + pack = 0 + prev_options = self._options + try: + self._options = {'override': override, + 'packed': pack, + 'dllexport': dllexport} + self._internal_parse(csource) + finally: + self._options = prev_options + + def _internal_parse(self, csource): + ast, macros, csource = self._parse(csource) + # add the macros + self._process_macros(macros) + # find the first "__dotdotdot__" and use that as a separator + # between the repeated typedefs and the real csource + iterator = iter(ast.ext) + for decl in iterator: + if decl.name == '__dotdotdot__': + break + else: + assert 0 + current_decl = None + # + try: + self._inside_extern_python = '__cffi_extern_python_stop' + for decl in iterator: + current_decl = decl + if isinstance(decl, pycparser.c_ast.Decl): + self._parse_decl(decl) + elif isinstance(decl, pycparser.c_ast.Typedef): + if not decl.name: + raise CDefError("typedef does not declare any name", + decl) + quals = 0 + if (isinstance(decl.type.type, pycparser.c_ast.IdentifierType) and + decl.type.type.names[-1].startswith('__dotdotdot')): + realtype = self._get_unknown_type(decl) + elif (isinstance(decl.type, pycparser.c_ast.PtrDecl) and + isinstance(decl.type.type, pycparser.c_ast.TypeDecl) and + isinstance(decl.type.type.type, + pycparser.c_ast.IdentifierType) and + decl.type.type.type.names[-1].startswith('__dotdotdot')): + realtype = self._get_unknown_ptr_type(decl) + else: + realtype, quals = self._get_type_and_quals( + decl.type, name=decl.name, partial_length_ok=True, + typedef_example="*(%s *)0" % (decl.name,)) + self._declare('typedef ' + decl.name, realtype, quals=quals) + elif decl.__class__.__name__ == 'Pragma': + pass # skip pragma, only in pycparser 2.15 + else: + raise CDefError("unexpected <%s>: this construct is valid " + "C but not valid in cdef()" % + decl.__class__.__name__, decl) + except CDefError as e: + if len(e.args) == 1: + e.args = e.args + (current_decl,) + raise + except FFIError as e: + msg = self._convert_pycparser_error(e, csource) + if msg: + e.args = (e.args[0] + "\n *** Err: %s" % msg,) + raise + + def _add_constants(self, key, val): + if key in self._int_constants: + if self._int_constants[key] == val: + return # ignore identical double declarations + raise FFIError( + "multiple declarations of constant: %s" % (key,)) + self._int_constants[key] = val + + def _add_integer_constant(self, name, int_str): + int_str = int_str.lower().rstrip("ul") + neg = int_str.startswith('-') + if neg: + int_str = int_str[1:] + # "010" is not valid oct in py3 + if (int_str.startswith("0") and int_str != '0' + and not int_str.startswith("0x")): + int_str = "0o" + int_str[1:] + pyvalue = int(int_str, 0) + if neg: + pyvalue = -pyvalue + self._add_constants(name, pyvalue) + self._declare('macro ' + name, pyvalue) + + def _process_macros(self, macros): + for key, value in macros.items(): + value = value.strip() + if _r_int_literal.match(value): + self._add_integer_constant(key, value) + elif value == '...': + self._declare('macro ' + key, value) + else: + raise CDefError( + 'only supports one of the following syntax:\n' + ' #define %s ... (literally dot-dot-dot)\n' + ' #define %s NUMBER (with NUMBER an integer' + ' constant, decimal/hex/octal)\n' + 'got:\n' + ' #define %s %s' + % (key, key, key, value)) + + def _declare_function(self, tp, quals, decl): + tp = self._get_type_pointer(tp, quals) + if self._options.get('dllexport'): + tag = 'dllexport_python ' + elif self._inside_extern_python == '__cffi_extern_python_start': + tag = 'extern_python ' + elif self._inside_extern_python == '__cffi_extern_python_plus_c_start': + tag = 'extern_python_plus_c ' + else: + tag = 'function ' + self._declare(tag + decl.name, tp) + + def _parse_decl(self, decl): + node = decl.type + if isinstance(node, pycparser.c_ast.FuncDecl): + tp, quals = self._get_type_and_quals(node, name=decl.name) + assert isinstance(tp, model.RawFunctionType) + self._declare_function(tp, quals, decl) + else: + if isinstance(node, pycparser.c_ast.Struct): + self._get_struct_union_enum_type('struct', node) + elif isinstance(node, pycparser.c_ast.Union): + self._get_struct_union_enum_type('union', node) + elif isinstance(node, pycparser.c_ast.Enum): + self._get_struct_union_enum_type('enum', node) + elif not decl.name: + raise CDefError("construct does not declare any variable", + decl) + # + if decl.name: + tp, quals = self._get_type_and_quals(node, + partial_length_ok=True) + if tp.is_raw_function: + self._declare_function(tp, quals, decl) + elif (tp.is_integer_type() and + hasattr(decl, 'init') and + hasattr(decl.init, 'value') and + _r_int_literal.match(decl.init.value)): + self._add_integer_constant(decl.name, decl.init.value) + elif (tp.is_integer_type() and + isinstance(decl.init, pycparser.c_ast.UnaryOp) and + decl.init.op == '-' and + hasattr(decl.init.expr, 'value') and + _r_int_literal.match(decl.init.expr.value)): + self._add_integer_constant(decl.name, + '-' + decl.init.expr.value) + elif (tp is model.void_type and + decl.name.startswith('__cffi_extern_python_')): + # hack: `extern "Python"` in the C source is replaced + # with "void __cffi_extern_python_start;" and + # "void __cffi_extern_python_stop;" + self._inside_extern_python = decl.name + else: + if self._inside_extern_python !='__cffi_extern_python_stop': + raise CDefError( + "cannot declare constants or " + "variables with 'extern \"Python\"'") + if (quals & model.Q_CONST) and not tp.is_array_type: + self._declare('constant ' + decl.name, tp, quals=quals) + else: + _warn_for_non_extern_non_static_global_variable(decl) + self._declare('variable ' + decl.name, tp, quals=quals) + + def parse_type(self, cdecl): + return self.parse_type_and_quals(cdecl)[0] + + def parse_type_and_quals(self, cdecl): + ast, macros = self._parse('void __dummy(\n%s\n);' % cdecl)[:2] + assert not macros + exprnode = ast.ext[-1].type.args.params[0] + if isinstance(exprnode, pycparser.c_ast.ID): + raise CDefError("unknown identifier '%s'" % (exprnode.name,)) + return self._get_type_and_quals(exprnode.type) + + def _declare(self, name, obj, included=False, quals=0): + if name in self._declarations: + prevobj, prevquals = self._declarations[name] + if prevobj is obj and prevquals == quals: + return + if not self._options.get('override'): + raise FFIError( + "multiple declarations of %s (for interactive usage, " + "try cdef(xx, override=True))" % (name,)) + assert '__dotdotdot__' not in name.split() + self._declarations[name] = (obj, quals) + if included: + self._included_declarations.add(obj) + + def _extract_quals(self, type): + quals = 0 + if isinstance(type, (pycparser.c_ast.TypeDecl, + pycparser.c_ast.PtrDecl)): + if 'const' in type.quals: + quals |= model.Q_CONST + if 'volatile' in type.quals: + quals |= model.Q_VOLATILE + if 'restrict' in type.quals: + quals |= model.Q_RESTRICT + return quals + + def _get_type_pointer(self, type, quals, declname=None): + if isinstance(type, model.RawFunctionType): + return type.as_function_pointer() + if (isinstance(type, model.StructOrUnionOrEnum) and + type.name.startswith('$') and type.name[1:].isdigit() and + type.forcename is None and declname is not None): + return model.NamedPointerType(type, declname, quals) + return model.PointerType(type, quals) + + def _get_type_and_quals(self, typenode, name=None, partial_length_ok=False, + typedef_example=None): + # first, dereference typedefs, if we have it already parsed, we're good + if (isinstance(typenode, pycparser.c_ast.TypeDecl) and + isinstance(typenode.type, pycparser.c_ast.IdentifierType) and + len(typenode.type.names) == 1 and + ('typedef ' + typenode.type.names[0]) in self._declarations): + tp, quals = self._declarations['typedef ' + typenode.type.names[0]] + quals |= self._extract_quals(typenode) + return tp, quals + # + if isinstance(typenode, pycparser.c_ast.ArrayDecl): + # array type + if typenode.dim is None: + length = None + else: + length = self._parse_constant( + typenode.dim, partial_length_ok=partial_length_ok) + # a hack: in 'typedef int foo_t[...][...];', don't use '...' as + # the length but use directly the C expression that would be + # generated by recompiler.py. This lets the typedef be used in + # many more places within recompiler.py + if typedef_example is not None: + if length == '...': + length = '_cffi_array_len(%s)' % (typedef_example,) + typedef_example = "*" + typedef_example + # + tp, quals = self._get_type_and_quals(typenode.type, + partial_length_ok=partial_length_ok, + typedef_example=typedef_example) + return model.ArrayType(tp, length), quals + # + if isinstance(typenode, pycparser.c_ast.PtrDecl): + # pointer type + itemtype, itemquals = self._get_type_and_quals(typenode.type) + tp = self._get_type_pointer(itemtype, itemquals, declname=name) + quals = self._extract_quals(typenode) + return tp, quals + # + if isinstance(typenode, pycparser.c_ast.TypeDecl): + quals = self._extract_quals(typenode) + type = typenode.type + if isinstance(type, pycparser.c_ast.IdentifierType): + # assume a primitive type. get it from .names, but reduce + # synonyms to a single chosen combination + names = list(type.names) + if names != ['signed', 'char']: # keep this unmodified + prefixes = {} + while names: + name = names[0] + if name in ('short', 'long', 'signed', 'unsigned'): + prefixes[name] = prefixes.get(name, 0) + 1 + del names[0] + else: + break + # ignore the 'signed' prefix below, and reorder the others + newnames = [] + for prefix in ('unsigned', 'short', 'long'): + for i in range(prefixes.get(prefix, 0)): + newnames.append(prefix) + if not names: + names = ['int'] # implicitly + if names == ['int']: # but kill it if 'short' or 'long' + if 'short' in prefixes or 'long' in prefixes: + names = [] + names = newnames + names + ident = ' '.join(names) + if ident == 'void': + return model.void_type, quals + if ident == '__dotdotdot__': + raise FFIError(':%d: bad usage of "..."' % + typenode.coord.line) + tp0, quals0 = resolve_common_type(self, ident) + return tp0, (quals | quals0) + # + if isinstance(type, pycparser.c_ast.Struct): + # 'struct foobar' + tp = self._get_struct_union_enum_type('struct', type, name) + return tp, quals + # + if isinstance(type, pycparser.c_ast.Union): + # 'union foobar' + tp = self._get_struct_union_enum_type('union', type, name) + return tp, quals + # + if isinstance(type, pycparser.c_ast.Enum): + # 'enum foobar' + tp = self._get_struct_union_enum_type('enum', type, name) + return tp, quals + # + if isinstance(typenode, pycparser.c_ast.FuncDecl): + # a function type + return self._parse_function_type(typenode, name), 0 + # + # nested anonymous structs or unions end up here + if isinstance(typenode, pycparser.c_ast.Struct): + return self._get_struct_union_enum_type('struct', typenode, name, + nested=True), 0 + if isinstance(typenode, pycparser.c_ast.Union): + return self._get_struct_union_enum_type('union', typenode, name, + nested=True), 0 + # + raise FFIError(":%d: bad or unsupported type declaration" % + typenode.coord.line) + + def _parse_function_type(self, typenode, funcname=None): + params = list(getattr(typenode.args, 'params', [])) + for i, arg in enumerate(params): + if not hasattr(arg, 'type'): + raise CDefError("%s arg %d: unknown type '%s'" + " (if you meant to use the old C syntax of giving" + " untyped arguments, it is not supported)" + % (funcname or 'in expression', i + 1, + getattr(arg, 'name', '?'))) + ellipsis = ( + len(params) > 0 and + isinstance(params[-1].type, pycparser.c_ast.TypeDecl) and + isinstance(params[-1].type.type, + pycparser.c_ast.IdentifierType) and + params[-1].type.type.names == ['__dotdotdot__']) + if ellipsis: + params.pop() + if not params: + raise CDefError( + "%s: a function with only '(...)' as argument" + " is not correct C" % (funcname or 'in expression')) + args = [self._as_func_arg(*self._get_type_and_quals(argdeclnode.type)) + for argdeclnode in params] + if not ellipsis and args == [model.void_type]: + args = [] + result, quals = self._get_type_and_quals(typenode.type) + # the 'quals' on the result type are ignored. HACK: we absure them + # to detect __stdcall functions: we textually replace "__stdcall" + # with "volatile volatile const" above. + abi = None + if hasattr(typenode.type, 'quals'): # else, probable syntax error anyway + if typenode.type.quals[-3:] == ['volatile', 'volatile', 'const']: + abi = '__stdcall' + return model.RawFunctionType(tuple(args), result, ellipsis, abi) + + def _as_func_arg(self, type, quals): + if isinstance(type, model.ArrayType): + return model.PointerType(type.item, quals) + elif isinstance(type, model.RawFunctionType): + return type.as_function_pointer() + else: + return type + + def _get_struct_union_enum_type(self, kind, type, name=None, nested=False): + # First, a level of caching on the exact 'type' node of the AST. + # This is obscure, but needed because pycparser "unrolls" declarations + # such as "typedef struct { } foo_t, *foo_p" and we end up with + # an AST that is not a tree, but a DAG, with the "type" node of the + # two branches foo_t and foo_p of the trees being the same node. + # It's a bit silly but detecting "DAG-ness" in the AST tree seems + # to be the only way to distinguish this case from two independent + # structs. See test_struct_with_two_usages. + try: + return self._structnode2type[type] + except KeyError: + pass + # + # Note that this must handle parsing "struct foo" any number of + # times and always return the same StructType object. Additionally, + # one of these times (not necessarily the first), the fields of + # the struct can be specified with "struct foo { ...fields... }". + # If no name is given, then we have to create a new anonymous struct + # with no caching; in this case, the fields are either specified + # right now or never. + # + force_name = name + name = type.name + # + # get the type or create it if needed + if name is None: + # 'force_name' is used to guess a more readable name for + # anonymous structs, for the common case "typedef struct { } foo". + if force_name is not None: + explicit_name = '$%s' % force_name + else: + self._anonymous_counter += 1 + explicit_name = '$%d' % self._anonymous_counter + tp = None + else: + explicit_name = name + key = '%s %s' % (kind, name) + tp, _ = self._declarations.get(key, (None, None)) + # + if tp is None: + if kind == 'struct': + tp = model.StructType(explicit_name, None, None, None) + elif kind == 'union': + tp = model.UnionType(explicit_name, None, None, None) + elif kind == 'enum': + if explicit_name == '__dotdotdot__': + raise CDefError("Enums cannot be declared with ...") + tp = self._build_enum_type(explicit_name, type.values) + else: + raise AssertionError("kind = %r" % (kind,)) + if name is not None: + self._declare(key, tp) + else: + if kind == 'enum' and type.values is not None: + raise NotImplementedError( + "enum %s: the '{}' declaration should appear on the first " + "time the enum is mentioned, not later" % explicit_name) + if not tp.forcename: + tp.force_the_name(force_name) + if tp.forcename and '$' in tp.name: + self._declare('anonymous %s' % tp.forcename, tp) + # + self._structnode2type[type] = tp + # + # enums: done here + if kind == 'enum': + return tp + # + # is there a 'type.decls'? If yes, then this is the place in the + # C sources that declare the fields. If no, then just return the + # existing type, possibly still incomplete. + if type.decls is None: + return tp + # + if tp.fldnames is not None: + raise CDefError("duplicate declaration of struct %s" % name) + fldnames = [] + fldtypes = [] + fldbitsize = [] + fldquals = [] + for decl in type.decls: + if (isinstance(decl.type, pycparser.c_ast.IdentifierType) and + ''.join(decl.type.names) == '__dotdotdot__'): + # XXX pycparser is inconsistent: 'names' should be a list + # of strings, but is sometimes just one string. Use + # str.join() as a way to cope with both. + self._make_partial(tp, nested) + continue + if decl.bitsize is None: + bitsize = -1 + else: + bitsize = self._parse_constant(decl.bitsize) + self._partial_length = False + type, fqual = self._get_type_and_quals(decl.type, + partial_length_ok=True) + if self._partial_length: + self._make_partial(tp, nested) + if isinstance(type, model.StructType) and type.partial: + self._make_partial(tp, nested) + fldnames.append(decl.name or '') + fldtypes.append(type) + fldbitsize.append(bitsize) + fldquals.append(fqual) + tp.fldnames = tuple(fldnames) + tp.fldtypes = tuple(fldtypes) + tp.fldbitsize = tuple(fldbitsize) + tp.fldquals = tuple(fldquals) + if fldbitsize != [-1] * len(fldbitsize): + if isinstance(tp, model.StructType) and tp.partial: + raise NotImplementedError("%s: using both bitfields and '...;'" + % (tp,)) + tp.packed = self._options.get('packed') + if tp.completed: # must be re-completed: it is not opaque any more + tp.completed = 0 + self._recomplete.append(tp) + return tp + + def _make_partial(self, tp, nested): + if not isinstance(tp, model.StructOrUnion): + raise CDefError("%s cannot be partial" % (tp,)) + if not tp.has_c_name() and not nested: + raise NotImplementedError("%s is partial but has no C name" %(tp,)) + tp.partial = True + + def _parse_constant(self, exprnode, partial_length_ok=False): + # for now, limited to expressions that are an immediate number + # or positive/negative number + if isinstance(exprnode, pycparser.c_ast.Constant): + s = exprnode.value + if '0' <= s[0] <= '9': + s = s.rstrip('uUlL') + try: + if s.startswith('0'): + return int(s, 8) + else: + return int(s, 10) + except ValueError: + if len(s) > 1: + if s.lower()[0:2] == '0x': + return int(s, 16) + elif s.lower()[0:2] == '0b': + return int(s, 2) + raise CDefError("invalid constant %r" % (s,)) + elif s[0] == "'" and s[-1] == "'" and ( + len(s) == 3 or (len(s) == 4 and s[1] == "\\")): + return ord(s[-2]) + else: + raise CDefError("invalid constant %r" % (s,)) + # + if (isinstance(exprnode, pycparser.c_ast.UnaryOp) and + exprnode.op == '+'): + return self._parse_constant(exprnode.expr) + # + if (isinstance(exprnode, pycparser.c_ast.UnaryOp) and + exprnode.op == '-'): + return -self._parse_constant(exprnode.expr) + # load previously defined int constant + if (isinstance(exprnode, pycparser.c_ast.ID) and + exprnode.name in self._int_constants): + return self._int_constants[exprnode.name] + # + if (isinstance(exprnode, pycparser.c_ast.ID) and + exprnode.name == '__dotdotdotarray__'): + if partial_length_ok: + self._partial_length = True + return '...' + raise FFIError(":%d: unsupported '[...]' here, cannot derive " + "the actual array length in this context" + % exprnode.coord.line) + # + if isinstance(exprnode, pycparser.c_ast.BinaryOp): + left = self._parse_constant(exprnode.left) + right = self._parse_constant(exprnode.right) + if exprnode.op == '+': + return left + right + elif exprnode.op == '-': + return left - right + elif exprnode.op == '*': + return left * right + elif exprnode.op == '/': + return self._c_div(left, right) + elif exprnode.op == '%': + return left - self._c_div(left, right) * right + elif exprnode.op == '<<': + return left << right + elif exprnode.op == '>>': + return left >> right + elif exprnode.op == '&': + return left & right + elif exprnode.op == '|': + return left | right + elif exprnode.op == '^': + return left ^ right + # + raise FFIError(":%d: unsupported expression: expected a " + "simple numeric constant" % exprnode.coord.line) + + def _c_div(self, a, b): + result = a // b + if ((a < 0) ^ (b < 0)) and (a % b) != 0: + result += 1 + return result + + def _build_enum_type(self, explicit_name, decls): + if decls is not None: + partial = False + enumerators = [] + enumvalues = [] + nextenumvalue = 0 + for enum in decls.enumerators: + if _r_enum_dotdotdot.match(enum.name): + partial = True + continue + if enum.value is not None: + nextenumvalue = self._parse_constant(enum.value) + enumerators.append(enum.name) + enumvalues.append(nextenumvalue) + self._add_constants(enum.name, nextenumvalue) + nextenumvalue += 1 + enumerators = tuple(enumerators) + enumvalues = tuple(enumvalues) + tp = model.EnumType(explicit_name, enumerators, enumvalues) + tp.partial = partial + else: # opaque enum + tp = model.EnumType(explicit_name, (), ()) + return tp + + def include(self, other): + for name, (tp, quals) in other._declarations.items(): + if name.startswith('anonymous $enum_$'): + continue # fix for test_anonymous_enum_include + kind = name.split(' ', 1)[0] + if kind in ('struct', 'union', 'enum', 'anonymous', 'typedef'): + self._declare(name, tp, included=True, quals=quals) + for k, v in other._int_constants.items(): + self._add_constants(k, v) + + def _get_unknown_type(self, decl): + typenames = decl.type.type.names + if typenames == ['__dotdotdot__']: + return model.unknown_type(decl.name) + + if typenames == ['__dotdotdotint__']: + if self._uses_new_feature is None: + self._uses_new_feature = "'typedef int... %s'" % decl.name + return model.UnknownIntegerType(decl.name) + + if typenames == ['__dotdotdotfloat__']: + # note: not for 'long double' so far + if self._uses_new_feature is None: + self._uses_new_feature = "'typedef float... %s'" % decl.name + return model.UnknownFloatType(decl.name) + + raise FFIError(':%d: unsupported usage of "..." in typedef' + % decl.coord.line) + + def _get_unknown_ptr_type(self, decl): + if decl.type.type.type.names == ['__dotdotdot__']: + return model.unknown_ptr_type(decl.name) + raise FFIError(':%d: unsupported usage of "..." in typedef' + % decl.coord.line) diff --git a/WebCrawler/venv/Lib/site-packages/cffi/error.py b/WebCrawler/venv/Lib/site-packages/cffi/error.py new file mode 100644 index 0000000..0a27247 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/cffi/error.py @@ -0,0 +1,31 @@ + +class FFIError(Exception): + __module__ = 'cffi' + +class CDefError(Exception): + __module__ = 'cffi' + def __str__(self): + try: + current_decl = self.args[1] + filename = current_decl.coord.file + linenum = current_decl.coord.line + prefix = '%s:%d: ' % (filename, linenum) + except (AttributeError, TypeError, IndexError): + prefix = '' + return '%s%s' % (prefix, self.args[0]) + +class VerificationError(Exception): + """ An error raised when verification fails + """ + __module__ = 'cffi' + +class VerificationMissing(Exception): + """ An error raised when incomplete structures are passed into + cdef, but no verification has been done + """ + __module__ = 'cffi' + +class PkgConfigError(Exception): + """ An error raised for missing modules in pkg-config + """ + __module__ = 'cffi' diff --git a/WebCrawler/venv/Lib/site-packages/cffi/ffiplatform.py b/WebCrawler/venv/Lib/site-packages/cffi/ffiplatform.py new file mode 100644 index 0000000..8531346 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/cffi/ffiplatform.py @@ -0,0 +1,127 @@ +import sys, os +from .error import VerificationError + + +LIST_OF_FILE_NAMES = ['sources', 'include_dirs', 'library_dirs', + 'extra_objects', 'depends'] + +def get_extension(srcfilename, modname, sources=(), **kwds): + _hack_at_distutils() + from distutils.core import Extension + allsources = [srcfilename] + for src in sources: + allsources.append(os.path.normpath(src)) + return Extension(name=modname, sources=allsources, **kwds) + +def compile(tmpdir, ext, compiler_verbose=0, debug=None): + """Compile a C extension module using distutils.""" + + _hack_at_distutils() + saved_environ = os.environ.copy() + try: + outputfilename = _build(tmpdir, ext, compiler_verbose, debug) + outputfilename = os.path.abspath(outputfilename) + finally: + # workaround for a distutils bugs where some env vars can + # become longer and longer every time it is used + for key, value in saved_environ.items(): + if os.environ.get(key) != value: + os.environ[key] = value + return outputfilename + +def _build(tmpdir, ext, compiler_verbose=0, debug=None): + # XXX compact but horrible :-( + from distutils.core import Distribution + import distutils.errors, distutils.log + # + dist = Distribution({'ext_modules': [ext]}) + dist.parse_config_files() + options = dist.get_option_dict('build_ext') + if debug is None: + debug = sys.flags.debug + options['debug'] = ('ffiplatform', debug) + options['force'] = ('ffiplatform', True) + options['build_lib'] = ('ffiplatform', tmpdir) + options['build_temp'] = ('ffiplatform', tmpdir) + # + try: + old_level = distutils.log.set_threshold(0) or 0 + try: + distutils.log.set_verbosity(compiler_verbose) + dist.run_command('build_ext') + cmd_obj = dist.get_command_obj('build_ext') + [soname] = cmd_obj.get_outputs() + finally: + distutils.log.set_threshold(old_level) + except (distutils.errors.CompileError, + distutils.errors.LinkError) as e: + raise VerificationError('%s: %s' % (e.__class__.__name__, e)) + # + return soname + +try: + from os.path import samefile +except ImportError: + def samefile(f1, f2): + return os.path.abspath(f1) == os.path.abspath(f2) + +def maybe_relative_path(path): + if not os.path.isabs(path): + return path # already relative + dir = path + names = [] + while True: + prevdir = dir + dir, name = os.path.split(prevdir) + if dir == prevdir or not dir: + return path # failed to make it relative + names.append(name) + try: + if samefile(dir, os.curdir): + names.reverse() + return os.path.join(*names) + except OSError: + pass + +# ____________________________________________________________ + +try: + int_or_long = (int, long) + import cStringIO +except NameError: + int_or_long = int # Python 3 + import io as cStringIO + +def _flatten(x, f): + if isinstance(x, str): + f.write('%ds%s' % (len(x), x)) + elif isinstance(x, dict): + keys = sorted(x.keys()) + f.write('%dd' % len(keys)) + for key in keys: + _flatten(key, f) + _flatten(x[key], f) + elif isinstance(x, (list, tuple)): + f.write('%dl' % len(x)) + for value in x: + _flatten(value, f) + elif isinstance(x, int_or_long): + f.write('%di' % (x,)) + else: + raise TypeError( + "the keywords to verify() contains unsupported object %r" % (x,)) + +def flatten(x): + f = cStringIO.StringIO() + _flatten(x, f) + return f.getvalue() + +def _hack_at_distutils(): + # Windows-only workaround for some configurations: see + # https://bugs.python.org/issue23246 (Python 2.7 with + # a specific MS compiler suite download) + if sys.platform == "win32": + try: + import setuptools # for side-effects, patches distutils + except ImportError: + pass diff --git a/WebCrawler/venv/Lib/site-packages/cffi/lock.py b/WebCrawler/venv/Lib/site-packages/cffi/lock.py new file mode 100644 index 0000000..db91b71 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/cffi/lock.py @@ -0,0 +1,30 @@ +import sys + +if sys.version_info < (3,): + try: + from thread import allocate_lock + except ImportError: + from dummy_thread import allocate_lock +else: + try: + from _thread import allocate_lock + except ImportError: + from _dummy_thread import allocate_lock + + +##import sys +##l1 = allocate_lock + +##class allocate_lock(object): +## def __init__(self): +## self._real = l1() +## def __enter__(self): +## for i in range(4, 0, -1): +## print sys._getframe(i).f_code +## print +## return self._real.__enter__() +## def __exit__(self, *args): +## return self._real.__exit__(*args) +## def acquire(self, f): +## assert f is False +## return self._real.acquire(f) diff --git a/WebCrawler/venv/Lib/site-packages/cffi/model.py b/WebCrawler/venv/Lib/site-packages/cffi/model.py new file mode 100644 index 0000000..ad1c176 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/cffi/model.py @@ -0,0 +1,617 @@ +import types +import weakref + +from .lock import allocate_lock +from .error import CDefError, VerificationError, VerificationMissing + +# type qualifiers +Q_CONST = 0x01 +Q_RESTRICT = 0x02 +Q_VOLATILE = 0x04 + +def qualify(quals, replace_with): + if quals & Q_CONST: + replace_with = ' const ' + replace_with.lstrip() + if quals & Q_VOLATILE: + replace_with = ' volatile ' + replace_with.lstrip() + if quals & Q_RESTRICT: + # It seems that __restrict is supported by gcc and msvc. + # If you hit some different compiler, add a #define in + # _cffi_include.h for it (and in its copies, documented there) + replace_with = ' __restrict ' + replace_with.lstrip() + return replace_with + + +class BaseTypeByIdentity(object): + is_array_type = False + is_raw_function = False + + def get_c_name(self, replace_with='', context='a C file', quals=0): + result = self.c_name_with_marker + assert result.count('&') == 1 + # some logic duplication with ffi.getctype()... :-( + replace_with = replace_with.strip() + if replace_with: + if replace_with.startswith('*') and '&[' in result: + replace_with = '(%s)' % replace_with + elif not replace_with[0] in '[(': + replace_with = ' ' + replace_with + replace_with = qualify(quals, replace_with) + result = result.replace('&', replace_with) + if '$' in result: + raise VerificationError( + "cannot generate '%s' in %s: unknown type name" + % (self._get_c_name(), context)) + return result + + def _get_c_name(self): + return self.c_name_with_marker.replace('&', '') + + def has_c_name(self): + return '$' not in self._get_c_name() + + def is_integer_type(self): + return False + + def get_cached_btype(self, ffi, finishlist, can_delay=False): + try: + BType = ffi._cached_btypes[self] + except KeyError: + BType = self.build_backend_type(ffi, finishlist) + BType2 = ffi._cached_btypes.setdefault(self, BType) + assert BType2 is BType + return BType + + def __repr__(self): + return '<%s>' % (self._get_c_name(),) + + def _get_items(self): + return [(name, getattr(self, name)) for name in self._attrs_] + + +class BaseType(BaseTypeByIdentity): + + def __eq__(self, other): + return (self.__class__ == other.__class__ and + self._get_items() == other._get_items()) + + def __ne__(self, other): + return not self == other + + def __hash__(self): + return hash((self.__class__, tuple(self._get_items()))) + + +class VoidType(BaseType): + _attrs_ = () + + def __init__(self): + self.c_name_with_marker = 'void&' + + def build_backend_type(self, ffi, finishlist): + return global_cache(self, ffi, 'new_void_type') + +void_type = VoidType() + + +class BasePrimitiveType(BaseType): + def is_complex_type(self): + return False + + +class PrimitiveType(BasePrimitiveType): + _attrs_ = ('name',) + + ALL_PRIMITIVE_TYPES = { + 'char': 'c', + 'short': 'i', + 'int': 'i', + 'long': 'i', + 'long long': 'i', + 'signed char': 'i', + 'unsigned char': 'i', + 'unsigned short': 'i', + 'unsigned int': 'i', + 'unsigned long': 'i', + 'unsigned long long': 'i', + 'float': 'f', + 'double': 'f', + 'long double': 'f', + 'float _Complex': 'j', + 'double _Complex': 'j', + '_Bool': 'i', + # the following types are not primitive in the C sense + 'wchar_t': 'c', + 'char16_t': 'c', + 'char32_t': 'c', + 'int8_t': 'i', + 'uint8_t': 'i', + 'int16_t': 'i', + 'uint16_t': 'i', + 'int32_t': 'i', + 'uint32_t': 'i', + 'int64_t': 'i', + 'uint64_t': 'i', + 'int_least8_t': 'i', + 'uint_least8_t': 'i', + 'int_least16_t': 'i', + 'uint_least16_t': 'i', + 'int_least32_t': 'i', + 'uint_least32_t': 'i', + 'int_least64_t': 'i', + 'uint_least64_t': 'i', + 'int_fast8_t': 'i', + 'uint_fast8_t': 'i', + 'int_fast16_t': 'i', + 'uint_fast16_t': 'i', + 'int_fast32_t': 'i', + 'uint_fast32_t': 'i', + 'int_fast64_t': 'i', + 'uint_fast64_t': 'i', + 'intptr_t': 'i', + 'uintptr_t': 'i', + 'intmax_t': 'i', + 'uintmax_t': 'i', + 'ptrdiff_t': 'i', + 'size_t': 'i', + 'ssize_t': 'i', + } + + def __init__(self, name): + assert name in self.ALL_PRIMITIVE_TYPES + self.name = name + self.c_name_with_marker = name + '&' + + def is_char_type(self): + return self.ALL_PRIMITIVE_TYPES[self.name] == 'c' + def is_integer_type(self): + return self.ALL_PRIMITIVE_TYPES[self.name] == 'i' + def is_float_type(self): + return self.ALL_PRIMITIVE_TYPES[self.name] == 'f' + def is_complex_type(self): + return self.ALL_PRIMITIVE_TYPES[self.name] == 'j' + + def build_backend_type(self, ffi, finishlist): + return global_cache(self, ffi, 'new_primitive_type', self.name) + + +class UnknownIntegerType(BasePrimitiveType): + _attrs_ = ('name',) + + def __init__(self, name): + self.name = name + self.c_name_with_marker = name + '&' + + def is_integer_type(self): + return True + + def build_backend_type(self, ffi, finishlist): + raise NotImplementedError("integer type '%s' can only be used after " + "compilation" % self.name) + +class UnknownFloatType(BasePrimitiveType): + _attrs_ = ('name', ) + + def __init__(self, name): + self.name = name + self.c_name_with_marker = name + '&' + + def build_backend_type(self, ffi, finishlist): + raise NotImplementedError("float type '%s' can only be used after " + "compilation" % self.name) + + +class BaseFunctionType(BaseType): + _attrs_ = ('args', 'result', 'ellipsis', 'abi') + + def __init__(self, args, result, ellipsis, abi=None): + self.args = args + self.result = result + self.ellipsis = ellipsis + self.abi = abi + # + reprargs = [arg._get_c_name() for arg in self.args] + if self.ellipsis: + reprargs.append('...') + reprargs = reprargs or ['void'] + replace_with = self._base_pattern % (', '.join(reprargs),) + if abi is not None: + replace_with = replace_with[:1] + abi + ' ' + replace_with[1:] + self.c_name_with_marker = ( + self.result.c_name_with_marker.replace('&', replace_with)) + + +class RawFunctionType(BaseFunctionType): + # Corresponds to a C type like 'int(int)', which is the C type of + # a function, but not a pointer-to-function. The backend has no + # notion of such a type; it's used temporarily by parsing. + _base_pattern = '(&)(%s)' + is_raw_function = True + + def build_backend_type(self, ffi, finishlist): + raise CDefError("cannot render the type %r: it is a function " + "type, not a pointer-to-function type" % (self,)) + + def as_function_pointer(self): + return FunctionPtrType(self.args, self.result, self.ellipsis, self.abi) + + +class FunctionPtrType(BaseFunctionType): + _base_pattern = '(*&)(%s)' + + def build_backend_type(self, ffi, finishlist): + result = self.result.get_cached_btype(ffi, finishlist) + args = [] + for tp in self.args: + args.append(tp.get_cached_btype(ffi, finishlist)) + abi_args = () + if self.abi == "__stdcall": + if not self.ellipsis: # __stdcall ignored for variadic funcs + try: + abi_args = (ffi._backend.FFI_STDCALL,) + except AttributeError: + pass + return global_cache(self, ffi, 'new_function_type', + tuple(args), result, self.ellipsis, *abi_args) + + def as_raw_function(self): + return RawFunctionType(self.args, self.result, self.ellipsis, self.abi) + + +class PointerType(BaseType): + _attrs_ = ('totype', 'quals') + + def __init__(self, totype, quals=0): + self.totype = totype + self.quals = quals + extra = qualify(quals, " *&") + if totype.is_array_type: + extra = "(%s)" % (extra.lstrip(),) + self.c_name_with_marker = totype.c_name_with_marker.replace('&', extra) + + def build_backend_type(self, ffi, finishlist): + BItem = self.totype.get_cached_btype(ffi, finishlist, can_delay=True) + return global_cache(self, ffi, 'new_pointer_type', BItem) + +voidp_type = PointerType(void_type) + +def ConstPointerType(totype): + return PointerType(totype, Q_CONST) + +const_voidp_type = ConstPointerType(void_type) + + +class NamedPointerType(PointerType): + _attrs_ = ('totype', 'name') + + def __init__(self, totype, name, quals=0): + PointerType.__init__(self, totype, quals) + self.name = name + self.c_name_with_marker = name + '&' + + +class ArrayType(BaseType): + _attrs_ = ('item', 'length') + is_array_type = True + + def __init__(self, item, length): + self.item = item + self.length = length + # + if length is None: + brackets = '&[]' + elif length == '...': + brackets = '&[/*...*/]' + else: + brackets = '&[%s]' % length + self.c_name_with_marker = ( + self.item.c_name_with_marker.replace('&', brackets)) + + def length_is_unknown(self): + return isinstance(self.length, str) + + def resolve_length(self, newlength): + return ArrayType(self.item, newlength) + + def build_backend_type(self, ffi, finishlist): + if self.length_is_unknown(): + raise CDefError("cannot render the type %r: unknown length" % + (self,)) + self.item.get_cached_btype(ffi, finishlist) # force the item BType + BPtrItem = PointerType(self.item).get_cached_btype(ffi, finishlist) + return global_cache(self, ffi, 'new_array_type', BPtrItem, self.length) + +char_array_type = ArrayType(PrimitiveType('char'), None) + + +class StructOrUnionOrEnum(BaseTypeByIdentity): + _attrs_ = ('name',) + forcename = None + + def build_c_name_with_marker(self): + name = self.forcename or '%s %s' % (self.kind, self.name) + self.c_name_with_marker = name + '&' + + def force_the_name(self, forcename): + self.forcename = forcename + self.build_c_name_with_marker() + + def get_official_name(self): + assert self.c_name_with_marker.endswith('&') + return self.c_name_with_marker[:-1] + + +class StructOrUnion(StructOrUnionOrEnum): + fixedlayout = None + completed = 0 + partial = False + packed = 0 + + def __init__(self, name, fldnames, fldtypes, fldbitsize, fldquals=None): + self.name = name + self.fldnames = fldnames + self.fldtypes = fldtypes + self.fldbitsize = fldbitsize + self.fldquals = fldquals + self.build_c_name_with_marker() + + def anonymous_struct_fields(self): + if self.fldtypes is not None: + for name, type in zip(self.fldnames, self.fldtypes): + if name == '' and isinstance(type, StructOrUnion): + yield type + + def enumfields(self, expand_anonymous_struct_union=True): + fldquals = self.fldquals + if fldquals is None: + fldquals = (0,) * len(self.fldnames) + for name, type, bitsize, quals in zip(self.fldnames, self.fldtypes, + self.fldbitsize, fldquals): + if (name == '' and isinstance(type, StructOrUnion) + and expand_anonymous_struct_union): + # nested anonymous struct/union + for result in type.enumfields(): + yield result + else: + yield (name, type, bitsize, quals) + + def force_flatten(self): + # force the struct or union to have a declaration that lists + # directly all fields returned by enumfields(), flattening + # nested anonymous structs/unions. + names = [] + types = [] + bitsizes = [] + fldquals = [] + for name, type, bitsize, quals in self.enumfields(): + names.append(name) + types.append(type) + bitsizes.append(bitsize) + fldquals.append(quals) + self.fldnames = tuple(names) + self.fldtypes = tuple(types) + self.fldbitsize = tuple(bitsizes) + self.fldquals = tuple(fldquals) + + def get_cached_btype(self, ffi, finishlist, can_delay=False): + BType = StructOrUnionOrEnum.get_cached_btype(self, ffi, finishlist, + can_delay) + if not can_delay: + self.finish_backend_type(ffi, finishlist) + return BType + + def finish_backend_type(self, ffi, finishlist): + if self.completed: + if self.completed != 2: + raise NotImplementedError("recursive structure declaration " + "for '%s'" % (self.name,)) + return + BType = ffi._cached_btypes[self] + # + self.completed = 1 + # + if self.fldtypes is None: + pass # not completing it: it's an opaque struct + # + elif self.fixedlayout is None: + fldtypes = [tp.get_cached_btype(ffi, finishlist) + for tp in self.fldtypes] + lst = list(zip(self.fldnames, fldtypes, self.fldbitsize)) + extra_flags = () + if self.packed: + if self.packed == 1: + extra_flags = (8,) # SF_PACKED + else: + extra_flags = (0, self.packed) + ffi._backend.complete_struct_or_union(BType, lst, self, + -1, -1, *extra_flags) + # + else: + fldtypes = [] + fieldofs, fieldsize, totalsize, totalalignment = self.fixedlayout + for i in range(len(self.fldnames)): + fsize = fieldsize[i] + ftype = self.fldtypes[i] + # + if isinstance(ftype, ArrayType) and ftype.length_is_unknown(): + # fix the length to match the total size + BItemType = ftype.item.get_cached_btype(ffi, finishlist) + nlen, nrest = divmod(fsize, ffi.sizeof(BItemType)) + if nrest != 0: + self._verification_error( + "field '%s.%s' has a bogus size?" % ( + self.name, self.fldnames[i] or '{}')) + ftype = ftype.resolve_length(nlen) + self.fldtypes = (self.fldtypes[:i] + (ftype,) + + self.fldtypes[i+1:]) + # + BFieldType = ftype.get_cached_btype(ffi, finishlist) + if isinstance(ftype, ArrayType) and ftype.length is None: + assert fsize == 0 + else: + bitemsize = ffi.sizeof(BFieldType) + if bitemsize != fsize: + self._verification_error( + "field '%s.%s' is declared as %d bytes, but is " + "really %d bytes" % (self.name, + self.fldnames[i] or '{}', + bitemsize, fsize)) + fldtypes.append(BFieldType) + # + lst = list(zip(self.fldnames, fldtypes, self.fldbitsize, fieldofs)) + ffi._backend.complete_struct_or_union(BType, lst, self, + totalsize, totalalignment) + self.completed = 2 + + def _verification_error(self, msg): + raise VerificationError(msg) + + def check_not_partial(self): + if self.partial and self.fixedlayout is None: + raise VerificationMissing(self._get_c_name()) + + def build_backend_type(self, ffi, finishlist): + self.check_not_partial() + finishlist.append(self) + # + return global_cache(self, ffi, 'new_%s_type' % self.kind, + self.get_official_name(), key=self) + + +class StructType(StructOrUnion): + kind = 'struct' + + +class UnionType(StructOrUnion): + kind = 'union' + + +class EnumType(StructOrUnionOrEnum): + kind = 'enum' + partial = False + partial_resolved = False + + def __init__(self, name, enumerators, enumvalues, baseinttype=None): + self.name = name + self.enumerators = enumerators + self.enumvalues = enumvalues + self.baseinttype = baseinttype + self.build_c_name_with_marker() + + def force_the_name(self, forcename): + StructOrUnionOrEnum.force_the_name(self, forcename) + if self.forcename is None: + name = self.get_official_name() + self.forcename = '$' + name.replace(' ', '_') + + def check_not_partial(self): + if self.partial and not self.partial_resolved: + raise VerificationMissing(self._get_c_name()) + + def build_backend_type(self, ffi, finishlist): + self.check_not_partial() + base_btype = self.build_baseinttype(ffi, finishlist) + return global_cache(self, ffi, 'new_enum_type', + self.get_official_name(), + self.enumerators, self.enumvalues, + base_btype, key=self) + + def build_baseinttype(self, ffi, finishlist): + if self.baseinttype is not None: + return self.baseinttype.get_cached_btype(ffi, finishlist) + # + if self.enumvalues: + smallest_value = min(self.enumvalues) + largest_value = max(self.enumvalues) + else: + import warnings + try: + # XXX! The goal is to ensure that the warnings.warn() + # will not suppress the warning. We want to get it + # several times if we reach this point several times. + __warningregistry__.clear() + except NameError: + pass + warnings.warn("%r has no values explicitly defined; " + "guessing that it is equivalent to 'unsigned int'" + % self._get_c_name()) + smallest_value = largest_value = 0 + if smallest_value < 0: # needs a signed type + sign = 1 + candidate1 = PrimitiveType("int") + candidate2 = PrimitiveType("long") + else: + sign = 0 + candidate1 = PrimitiveType("unsigned int") + candidate2 = PrimitiveType("unsigned long") + btype1 = candidate1.get_cached_btype(ffi, finishlist) + btype2 = candidate2.get_cached_btype(ffi, finishlist) + size1 = ffi.sizeof(btype1) + size2 = ffi.sizeof(btype2) + if (smallest_value >= ((-1) << (8*size1-1)) and + largest_value < (1 << (8*size1-sign))): + return btype1 + if (smallest_value >= ((-1) << (8*size2-1)) and + largest_value < (1 << (8*size2-sign))): + return btype2 + raise CDefError("%s values don't all fit into either 'long' " + "or 'unsigned long'" % self._get_c_name()) + +def unknown_type(name, structname=None): + if structname is None: + structname = '$%s' % name + tp = StructType(structname, None, None, None) + tp.force_the_name(name) + tp.origin = "unknown_type" + return tp + +def unknown_ptr_type(name, structname=None): + if structname is None: + structname = '$$%s' % name + tp = StructType(structname, None, None, None) + return NamedPointerType(tp, name) + + +global_lock = allocate_lock() +_typecache_cffi_backend = weakref.WeakValueDictionary() + +def get_typecache(backend): + # returns _typecache_cffi_backend if backend is the _cffi_backend + # module, or type(backend).__typecache if backend is an instance of + # CTypesBackend (or some FakeBackend class during tests) + if isinstance(backend, types.ModuleType): + return _typecache_cffi_backend + with global_lock: + if not hasattr(type(backend), '__typecache'): + type(backend).__typecache = weakref.WeakValueDictionary() + return type(backend).__typecache + +def global_cache(srctype, ffi, funcname, *args, **kwds): + key = kwds.pop('key', (funcname, args)) + assert not kwds + try: + return ffi._typecache[key] + except KeyError: + pass + try: + res = getattr(ffi._backend, funcname)(*args) + except NotImplementedError as e: + raise NotImplementedError("%s: %r: %s" % (funcname, srctype, e)) + # note that setdefault() on WeakValueDictionary is not atomic + # and contains a rare bug (http://bugs.python.org/issue19542); + # we have to use a lock and do it ourselves + cache = ffi._typecache + with global_lock: + res1 = cache.get(key) + if res1 is None: + cache[key] = res + return res + else: + return res1 + +def pointer_cache(ffi, BType): + return global_cache('?', ffi, 'new_pointer_type', BType) + +def attach_exception_info(e, name): + if e.args and type(e.args[0]) is str: + e.args = ('%s: %s' % (name, e.args[0]),) + e.args[1:] diff --git a/WebCrawler/venv/Lib/site-packages/cffi/parse_c_type.h b/WebCrawler/venv/Lib/site-packages/cffi/parse_c_type.h new file mode 100644 index 0000000..84e4ef8 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/cffi/parse_c_type.h @@ -0,0 +1,181 @@ + +/* This part is from file 'cffi/parse_c_type.h'. It is copied at the + beginning of C sources generated by CFFI's ffi.set_source(). */ + +typedef void *_cffi_opcode_t; + +#define _CFFI_OP(opcode, arg) (_cffi_opcode_t)(opcode | (((uintptr_t)(arg)) << 8)) +#define _CFFI_GETOP(cffi_opcode) ((unsigned char)(uintptr_t)cffi_opcode) +#define _CFFI_GETARG(cffi_opcode) (((intptr_t)cffi_opcode) >> 8) + +#define _CFFI_OP_PRIMITIVE 1 +#define _CFFI_OP_POINTER 3 +#define _CFFI_OP_ARRAY 5 +#define _CFFI_OP_OPEN_ARRAY 7 +#define _CFFI_OP_STRUCT_UNION 9 +#define _CFFI_OP_ENUM 11 +#define _CFFI_OP_FUNCTION 13 +#define _CFFI_OP_FUNCTION_END 15 +#define _CFFI_OP_NOOP 17 +#define _CFFI_OP_BITFIELD 19 +#define _CFFI_OP_TYPENAME 21 +#define _CFFI_OP_CPYTHON_BLTN_V 23 // varargs +#define _CFFI_OP_CPYTHON_BLTN_N 25 // noargs +#define _CFFI_OP_CPYTHON_BLTN_O 27 // O (i.e. a single arg) +#define _CFFI_OP_CONSTANT 29 +#define _CFFI_OP_CONSTANT_INT 31 +#define _CFFI_OP_GLOBAL_VAR 33 +#define _CFFI_OP_DLOPEN_FUNC 35 +#define _CFFI_OP_DLOPEN_CONST 37 +#define _CFFI_OP_GLOBAL_VAR_F 39 +#define _CFFI_OP_EXTERN_PYTHON 41 + +#define _CFFI_PRIM_VOID 0 +#define _CFFI_PRIM_BOOL 1 +#define _CFFI_PRIM_CHAR 2 +#define _CFFI_PRIM_SCHAR 3 +#define _CFFI_PRIM_UCHAR 4 +#define _CFFI_PRIM_SHORT 5 +#define _CFFI_PRIM_USHORT 6 +#define _CFFI_PRIM_INT 7 +#define _CFFI_PRIM_UINT 8 +#define _CFFI_PRIM_LONG 9 +#define _CFFI_PRIM_ULONG 10 +#define _CFFI_PRIM_LONGLONG 11 +#define _CFFI_PRIM_ULONGLONG 12 +#define _CFFI_PRIM_FLOAT 13 +#define _CFFI_PRIM_DOUBLE 14 +#define _CFFI_PRIM_LONGDOUBLE 15 + +#define _CFFI_PRIM_WCHAR 16 +#define _CFFI_PRIM_INT8 17 +#define _CFFI_PRIM_UINT8 18 +#define _CFFI_PRIM_INT16 19 +#define _CFFI_PRIM_UINT16 20 +#define _CFFI_PRIM_INT32 21 +#define _CFFI_PRIM_UINT32 22 +#define _CFFI_PRIM_INT64 23 +#define _CFFI_PRIM_UINT64 24 +#define _CFFI_PRIM_INTPTR 25 +#define _CFFI_PRIM_UINTPTR 26 +#define _CFFI_PRIM_PTRDIFF 27 +#define _CFFI_PRIM_SIZE 28 +#define _CFFI_PRIM_SSIZE 29 +#define _CFFI_PRIM_INT_LEAST8 30 +#define _CFFI_PRIM_UINT_LEAST8 31 +#define _CFFI_PRIM_INT_LEAST16 32 +#define _CFFI_PRIM_UINT_LEAST16 33 +#define _CFFI_PRIM_INT_LEAST32 34 +#define _CFFI_PRIM_UINT_LEAST32 35 +#define _CFFI_PRIM_INT_LEAST64 36 +#define _CFFI_PRIM_UINT_LEAST64 37 +#define _CFFI_PRIM_INT_FAST8 38 +#define _CFFI_PRIM_UINT_FAST8 39 +#define _CFFI_PRIM_INT_FAST16 40 +#define _CFFI_PRIM_UINT_FAST16 41 +#define _CFFI_PRIM_INT_FAST32 42 +#define _CFFI_PRIM_UINT_FAST32 43 +#define _CFFI_PRIM_INT_FAST64 44 +#define _CFFI_PRIM_UINT_FAST64 45 +#define _CFFI_PRIM_INTMAX 46 +#define _CFFI_PRIM_UINTMAX 47 +#define _CFFI_PRIM_FLOATCOMPLEX 48 +#define _CFFI_PRIM_DOUBLECOMPLEX 49 +#define _CFFI_PRIM_CHAR16 50 +#define _CFFI_PRIM_CHAR32 51 + +#define _CFFI__NUM_PRIM 52 +#define _CFFI__UNKNOWN_PRIM (-1) +#define _CFFI__UNKNOWN_FLOAT_PRIM (-2) +#define _CFFI__UNKNOWN_LONG_DOUBLE (-3) + +#define _CFFI__IO_FILE_STRUCT (-1) + + +struct _cffi_global_s { + const char *name; + void *address; + _cffi_opcode_t type_op; + void *size_or_direct_fn; // OP_GLOBAL_VAR: size, or 0 if unknown + // OP_CPYTHON_BLTN_*: addr of direct function +}; + +struct _cffi_getconst_s { + unsigned long long value; + const struct _cffi_type_context_s *ctx; + int gindex; +}; + +struct _cffi_struct_union_s { + const char *name; + int type_index; // -> _cffi_types, on a OP_STRUCT_UNION + int flags; // _CFFI_F_* flags below + size_t size; + int alignment; + int first_field_index; // -> _cffi_fields array + int num_fields; +}; +#define _CFFI_F_UNION 0x01 // is a union, not a struct +#define _CFFI_F_CHECK_FIELDS 0x02 // complain if fields are not in the + // "standard layout" or if some are missing +#define _CFFI_F_PACKED 0x04 // for CHECK_FIELDS, assume a packed struct +#define _CFFI_F_EXTERNAL 0x08 // in some other ffi.include() +#define _CFFI_F_OPAQUE 0x10 // opaque + +struct _cffi_field_s { + const char *name; + size_t field_offset; + size_t field_size; + _cffi_opcode_t field_type_op; +}; + +struct _cffi_enum_s { + const char *name; + int type_index; // -> _cffi_types, on a OP_ENUM + int type_prim; // _CFFI_PRIM_xxx + const char *enumerators; // comma-delimited string +}; + +struct _cffi_typename_s { + const char *name; + int type_index; /* if opaque, points to a possibly artificial + OP_STRUCT which is itself opaque */ +}; + +struct _cffi_type_context_s { + _cffi_opcode_t *types; + const struct _cffi_global_s *globals; + const struct _cffi_field_s *fields; + const struct _cffi_struct_union_s *struct_unions; + const struct _cffi_enum_s *enums; + const struct _cffi_typename_s *typenames; + int num_globals; + int num_struct_unions; + int num_enums; + int num_typenames; + const char *const *includes; + int num_types; + int flags; /* future extension */ +}; + +struct _cffi_parse_info_s { + const struct _cffi_type_context_s *ctx; + _cffi_opcode_t *output; + unsigned int output_size; + size_t error_location; + const char *error_message; +}; + +struct _cffi_externpy_s { + const char *name; + size_t size_of_result; + void *reserved1, *reserved2; +}; + +#ifdef _CFFI_INTERNAL +static int parse_c_type(struct _cffi_parse_info_s *info, const char *input); +static int search_in_globals(const struct _cffi_type_context_s *ctx, + const char *search, size_t search_len); +static int search_in_struct_unions(const struct _cffi_type_context_s *ctx, + const char *search, size_t search_len); +#endif diff --git a/WebCrawler/venv/Lib/site-packages/cffi/pkgconfig.py b/WebCrawler/venv/Lib/site-packages/cffi/pkgconfig.py new file mode 100644 index 0000000..5c93f15 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/cffi/pkgconfig.py @@ -0,0 +1,121 @@ +# pkg-config, https://www.freedesktop.org/wiki/Software/pkg-config/ integration for cffi +import sys, os, subprocess + +from .error import PkgConfigError + + +def merge_flags(cfg1, cfg2): + """Merge values from cffi config flags cfg2 to cf1 + + Example: + merge_flags({"libraries": ["one"]}, {"libraries": ["two"]}) + {"libraries": ["one", "two"]} + """ + for key, value in cfg2.items(): + if key not in cfg1: + cfg1[key] = value + else: + if not isinstance(cfg1[key], list): + raise TypeError("cfg1[%r] should be a list of strings" % (key,)) + if not isinstance(value, list): + raise TypeError("cfg2[%r] should be a list of strings" % (key,)) + cfg1[key].extend(value) + return cfg1 + + +def call(libname, flag, encoding=sys.getfilesystemencoding()): + """Calls pkg-config and returns the output if found + """ + a = ["pkg-config", "--print-errors"] + a.append(flag) + a.append(libname) + try: + pc = subprocess.Popen(a, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + except EnvironmentError as e: + raise PkgConfigError("cannot run pkg-config: %s" % (str(e).strip(),)) + + bout, berr = pc.communicate() + if pc.returncode != 0: + try: + berr = berr.decode(encoding) + except Exception: + pass + raise PkgConfigError(berr.strip()) + + if sys.version_info >= (3,) and not isinstance(bout, str): # Python 3.x + try: + bout = bout.decode(encoding) + except UnicodeDecodeError: + raise PkgConfigError("pkg-config %s %s returned bytes that cannot " + "be decoded with encoding %r:\n%r" % + (flag, libname, encoding, bout)) + + if os.altsep != '\\' and '\\' in bout: + raise PkgConfigError("pkg-config %s %s returned an unsupported " + "backslash-escaped output:\n%r" % + (flag, libname, bout)) + return bout + + +def flags_from_pkgconfig(libs): + r"""Return compiler line flags for FFI.set_source based on pkg-config output + + Usage + ... + ffibuilder.set_source("_foo", pkgconfig = ["libfoo", "libbar >= 1.8.3"]) + + If pkg-config is installed on build machine, then arguments include_dirs, + library_dirs, libraries, define_macros, extra_compile_args and + extra_link_args are extended with an output of pkg-config for libfoo and + libbar. + + Raises PkgConfigError in case the pkg-config call fails. + """ + + def get_include_dirs(string): + return [x[2:] for x in string.split() if x.startswith("-I")] + + def get_library_dirs(string): + return [x[2:] for x in string.split() if x.startswith("-L")] + + def get_libraries(string): + return [x[2:] for x in string.split() if x.startswith("-l")] + + # convert -Dfoo=bar to list of tuples [("foo", "bar")] expected by distutils + def get_macros(string): + def _macro(x): + x = x[2:] # drop "-D" + if '=' in x: + return tuple(x.split("=", 1)) # "-Dfoo=bar" => ("foo", "bar") + else: + return (x, None) # "-Dfoo" => ("foo", None) + return [_macro(x) for x in string.split() if x.startswith("-D")] + + def get_other_cflags(string): + return [x for x in string.split() if not x.startswith("-I") and + not x.startswith("-D")] + + def get_other_libs(string): + return [x for x in string.split() if not x.startswith("-L") and + not x.startswith("-l")] + + # return kwargs for given libname + def kwargs(libname): + fse = sys.getfilesystemencoding() + all_cflags = call(libname, "--cflags") + all_libs = call(libname, "--libs") + return { + "include_dirs": get_include_dirs(all_cflags), + "library_dirs": get_library_dirs(all_libs), + "libraries": get_libraries(all_libs), + "define_macros": get_macros(all_cflags), + "extra_compile_args": get_other_cflags(all_cflags), + "extra_link_args": get_other_libs(all_libs), + } + + # merge all arguments together + ret = {} + for libname in libs: + lib_flags = kwargs(libname) + merge_flags(ret, lib_flags) + return ret diff --git a/WebCrawler/venv/Lib/site-packages/cffi/recompiler.py b/WebCrawler/venv/Lib/site-packages/cffi/recompiler.py new file mode 100644 index 0000000..86b37d7 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/cffi/recompiler.py @@ -0,0 +1,1581 @@ +import os, sys, io +from . import ffiplatform, model +from .error import VerificationError +from .cffi_opcode import * + +VERSION_BASE = 0x2601 +VERSION_EMBEDDED = 0x2701 +VERSION_CHAR16CHAR32 = 0x2801 + +USE_LIMITED_API = (sys.platform != 'win32' or sys.version_info < (3, 0) or + sys.version_info >= (3, 5)) + + +class GlobalExpr: + def __init__(self, name, address, type_op, size=0, check_value=0): + self.name = name + self.address = address + self.type_op = type_op + self.size = size + self.check_value = check_value + + def as_c_expr(self): + return ' { "%s", (void *)%s, %s, (void *)%s },' % ( + self.name, self.address, self.type_op.as_c_expr(), self.size) + + def as_python_expr(self): + return "b'%s%s',%d" % (self.type_op.as_python_bytes(), self.name, + self.check_value) + +class FieldExpr: + def __init__(self, name, field_offset, field_size, fbitsize, field_type_op): + self.name = name + self.field_offset = field_offset + self.field_size = field_size + self.fbitsize = fbitsize + self.field_type_op = field_type_op + + def as_c_expr(self): + spaces = " " * len(self.name) + return (' { "%s", %s,\n' % (self.name, self.field_offset) + + ' %s %s,\n' % (spaces, self.field_size) + + ' %s %s },' % (spaces, self.field_type_op.as_c_expr())) + + def as_python_expr(self): + raise NotImplementedError + + def as_field_python_expr(self): + if self.field_type_op.op == OP_NOOP: + size_expr = '' + elif self.field_type_op.op == OP_BITFIELD: + size_expr = format_four_bytes(self.fbitsize) + else: + raise NotImplementedError + return "b'%s%s%s'" % (self.field_type_op.as_python_bytes(), + size_expr, + self.name) + +class StructUnionExpr: + def __init__(self, name, type_index, flags, size, alignment, comment, + first_field_index, c_fields): + self.name = name + self.type_index = type_index + self.flags = flags + self.size = size + self.alignment = alignment + self.comment = comment + self.first_field_index = first_field_index + self.c_fields = c_fields + + def as_c_expr(self): + return (' { "%s", %d, %s,' % (self.name, self.type_index, self.flags) + + '\n %s, %s, ' % (self.size, self.alignment) + + '%d, %d ' % (self.first_field_index, len(self.c_fields)) + + ('/* %s */ ' % self.comment if self.comment else '') + + '},') + + def as_python_expr(self): + flags = eval(self.flags, G_FLAGS) + fields_expr = [c_field.as_field_python_expr() + for c_field in self.c_fields] + return "(b'%s%s%s',%s)" % ( + format_four_bytes(self.type_index), + format_four_bytes(flags), + self.name, + ','.join(fields_expr)) + +class EnumExpr: + def __init__(self, name, type_index, size, signed, allenums): + self.name = name + self.type_index = type_index + self.size = size + self.signed = signed + self.allenums = allenums + + def as_c_expr(self): + return (' { "%s", %d, _cffi_prim_int(%s, %s),\n' + ' "%s" },' % (self.name, self.type_index, + self.size, self.signed, self.allenums)) + + def as_python_expr(self): + prim_index = { + (1, 0): PRIM_UINT8, (1, 1): PRIM_INT8, + (2, 0): PRIM_UINT16, (2, 1): PRIM_INT16, + (4, 0): PRIM_UINT32, (4, 1): PRIM_INT32, + (8, 0): PRIM_UINT64, (8, 1): PRIM_INT64, + }[self.size, self.signed] + return "b'%s%s%s\\x00%s'" % (format_four_bytes(self.type_index), + format_four_bytes(prim_index), + self.name, self.allenums) + +class TypenameExpr: + def __init__(self, name, type_index): + self.name = name + self.type_index = type_index + + def as_c_expr(self): + return ' { "%s", %d },' % (self.name, self.type_index) + + def as_python_expr(self): + return "b'%s%s'" % (format_four_bytes(self.type_index), self.name) + + +# ____________________________________________________________ + + +class Recompiler: + _num_externpy = 0 + + def __init__(self, ffi, module_name, target_is_python=False): + self.ffi = ffi + self.module_name = module_name + self.target_is_python = target_is_python + self._version = VERSION_BASE + + def needs_version(self, ver): + self._version = max(self._version, ver) + + def collect_type_table(self): + self._typesdict = {} + self._generate("collecttype") + # + all_decls = sorted(self._typesdict, key=str) + # + # prepare all FUNCTION bytecode sequences first + self.cffi_types = [] + for tp in all_decls: + if tp.is_raw_function: + assert self._typesdict[tp] is None + self._typesdict[tp] = len(self.cffi_types) + self.cffi_types.append(tp) # placeholder + for tp1 in tp.args: + assert isinstance(tp1, (model.VoidType, + model.BasePrimitiveType, + model.PointerType, + model.StructOrUnionOrEnum, + model.FunctionPtrType)) + if self._typesdict[tp1] is None: + self._typesdict[tp1] = len(self.cffi_types) + self.cffi_types.append(tp1) # placeholder + self.cffi_types.append('END') # placeholder + # + # prepare all OTHER bytecode sequences + for tp in all_decls: + if not tp.is_raw_function and self._typesdict[tp] is None: + self._typesdict[tp] = len(self.cffi_types) + self.cffi_types.append(tp) # placeholder + if tp.is_array_type and tp.length is not None: + self.cffi_types.append('LEN') # placeholder + assert None not in self._typesdict.values() + # + # collect all structs and unions and enums + self._struct_unions = {} + self._enums = {} + for tp in all_decls: + if isinstance(tp, model.StructOrUnion): + self._struct_unions[tp] = None + elif isinstance(tp, model.EnumType): + self._enums[tp] = None + for i, tp in enumerate(sorted(self._struct_unions, + key=lambda tp: tp.name)): + self._struct_unions[tp] = i + for i, tp in enumerate(sorted(self._enums, + key=lambda tp: tp.name)): + self._enums[tp] = i + # + # emit all bytecode sequences now + for tp in all_decls: + method = getattr(self, '_emit_bytecode_' + tp.__class__.__name__) + method(tp, self._typesdict[tp]) + # + # consistency check + for op in self.cffi_types: + assert isinstance(op, CffiOp) + self.cffi_types = tuple(self.cffi_types) # don't change any more + + def _enum_fields(self, tp): + # When producing C, expand all anonymous struct/union fields. + # That's necessary to have C code checking the offsets of the + # individual fields contained in them. When producing Python, + # don't do it and instead write it like it is, with the + # corresponding fields having an empty name. Empty names are + # recognized at runtime when we import the generated Python + # file. + expand_anonymous_struct_union = not self.target_is_python + return tp.enumfields(expand_anonymous_struct_union) + + def _do_collect_type(self, tp): + if not isinstance(tp, model.BaseTypeByIdentity): + if isinstance(tp, tuple): + for x in tp: + self._do_collect_type(x) + return + if tp not in self._typesdict: + self._typesdict[tp] = None + if isinstance(tp, model.FunctionPtrType): + self._do_collect_type(tp.as_raw_function()) + elif isinstance(tp, model.StructOrUnion): + if tp.fldtypes is not None and ( + tp not in self.ffi._parser._included_declarations): + for name1, tp1, _, _ in self._enum_fields(tp): + self._do_collect_type(self._field_type(tp, name1, tp1)) + else: + for _, x in tp._get_items(): + self._do_collect_type(x) + + def _generate(self, step_name): + lst = self.ffi._parser._declarations.items() + for name, (tp, quals) in sorted(lst): + kind, realname = name.split(' ', 1) + try: + method = getattr(self, '_generate_cpy_%s_%s' % (kind, + step_name)) + except AttributeError: + raise VerificationError( + "not implemented in recompile(): %r" % name) + try: + self._current_quals = quals + method(tp, realname) + except Exception as e: + model.attach_exception_info(e, name) + raise + + # ---------- + + ALL_STEPS = ["global", "field", "struct_union", "enum", "typename"] + + def collect_step_tables(self): + # collect the declarations for '_cffi_globals', '_cffi_typenames', etc. + self._lsts = {} + for step_name in self.ALL_STEPS: + self._lsts[step_name] = [] + self._seen_struct_unions = set() + self._generate("ctx") + self._add_missing_struct_unions() + # + for step_name in self.ALL_STEPS: + lst = self._lsts[step_name] + if step_name != "field": + lst.sort(key=lambda entry: entry.name) + self._lsts[step_name] = tuple(lst) # don't change any more + # + # check for a possible internal inconsistency: _cffi_struct_unions + # should have been generated with exactly self._struct_unions + lst = self._lsts["struct_union"] + for tp, i in self._struct_unions.items(): + assert i < len(lst) + assert lst[i].name == tp.name + assert len(lst) == len(self._struct_unions) + # same with enums + lst = self._lsts["enum"] + for tp, i in self._enums.items(): + assert i < len(lst) + assert lst[i].name == tp.name + assert len(lst) == len(self._enums) + + # ---------- + + def _prnt(self, what=''): + self._f.write(what + '\n') + + def write_source_to_f(self, f, preamble): + if self.target_is_python: + assert preamble is None + self.write_py_source_to_f(f) + else: + assert preamble is not None + self.write_c_source_to_f(f, preamble) + + def _rel_readlines(self, filename): + g = open(os.path.join(os.path.dirname(__file__), filename), 'r') + lines = g.readlines() + g.close() + return lines + + def write_c_source_to_f(self, f, preamble): + self._f = f + prnt = self._prnt + if self.ffi._embedding is not None: + prnt('#define _CFFI_USE_EMBEDDING') + if not USE_LIMITED_API: + prnt('#define _CFFI_NO_LIMITED_API') + # + # first the '#include' (actually done by inlining the file's content) + lines = self._rel_readlines('_cffi_include.h') + i = lines.index('#include "parse_c_type.h"\n') + lines[i:i+1] = self._rel_readlines('parse_c_type.h') + prnt(''.join(lines)) + # + # if we have ffi._embedding != None, we give it here as a macro + # and include an extra file + base_module_name = self.module_name.split('.')[-1] + if self.ffi._embedding is not None: + prnt('#define _CFFI_MODULE_NAME "%s"' % (self.module_name,)) + prnt('static const char _CFFI_PYTHON_STARTUP_CODE[] = {') + self._print_string_literal_in_array(self.ffi._embedding) + prnt('0 };') + prnt('#ifdef PYPY_VERSION') + prnt('# define _CFFI_PYTHON_STARTUP_FUNC _cffi_pypyinit_%s' % ( + base_module_name,)) + prnt('#elif PY_MAJOR_VERSION >= 3') + prnt('# define _CFFI_PYTHON_STARTUP_FUNC PyInit_%s' % ( + base_module_name,)) + prnt('#else') + prnt('# define _CFFI_PYTHON_STARTUP_FUNC init%s' % ( + base_module_name,)) + prnt('#endif') + lines = self._rel_readlines('_embedding.h') + i = lines.index('#include "_cffi_errors.h"\n') + lines[i:i+1] = self._rel_readlines('_cffi_errors.h') + prnt(''.join(lines)) + self.needs_version(VERSION_EMBEDDED) + # + # then paste the C source given by the user, verbatim. + prnt('/************************************************************/') + prnt() + prnt(preamble) + prnt() + prnt('/************************************************************/') + prnt() + # + # the declaration of '_cffi_types' + prnt('static void *_cffi_types[] = {') + typeindex2type = dict([(i, tp) for (tp, i) in self._typesdict.items()]) + for i, op in enumerate(self.cffi_types): + comment = '' + if i in typeindex2type: + comment = ' // ' + typeindex2type[i]._get_c_name() + prnt('/* %2d */ %s,%s' % (i, op.as_c_expr(), comment)) + if not self.cffi_types: + prnt(' 0') + prnt('};') + prnt() + # + # call generate_cpy_xxx_decl(), for every xxx found from + # ffi._parser._declarations. This generates all the functions. + self._seen_constants = set() + self._generate("decl") + # + # the declaration of '_cffi_globals' and '_cffi_typenames' + nums = {} + for step_name in self.ALL_STEPS: + lst = self._lsts[step_name] + nums[step_name] = len(lst) + if nums[step_name] > 0: + prnt('static const struct _cffi_%s_s _cffi_%ss[] = {' % ( + step_name, step_name)) + for entry in lst: + prnt(entry.as_c_expr()) + prnt('};') + prnt() + # + # the declaration of '_cffi_includes' + if self.ffi._included_ffis: + prnt('static const char * const _cffi_includes[] = {') + for ffi_to_include in self.ffi._included_ffis: + try: + included_module_name, included_source = ( + ffi_to_include._assigned_source[:2]) + except AttributeError: + raise VerificationError( + "ffi object %r includes %r, but the latter has not " + "been prepared with set_source()" % ( + self.ffi, ffi_to_include,)) + if included_source is None: + raise VerificationError( + "not implemented yet: ffi.include() of a Python-based " + "ffi inside a C-based ffi") + prnt(' "%s",' % (included_module_name,)) + prnt(' NULL') + prnt('};') + prnt() + # + # the declaration of '_cffi_type_context' + prnt('static const struct _cffi_type_context_s _cffi_type_context = {') + prnt(' _cffi_types,') + for step_name in self.ALL_STEPS: + if nums[step_name] > 0: + prnt(' _cffi_%ss,' % step_name) + else: + prnt(' NULL, /* no %ss */' % step_name) + for step_name in self.ALL_STEPS: + if step_name != "field": + prnt(' %d, /* num_%ss */' % (nums[step_name], step_name)) + if self.ffi._included_ffis: + prnt(' _cffi_includes,') + else: + prnt(' NULL, /* no includes */') + prnt(' %d, /* num_types */' % (len(self.cffi_types),)) + flags = 0 + if self._num_externpy: + flags |= 1 # set to mean that we use extern "Python" + prnt(' %d, /* flags */' % flags) + prnt('};') + prnt() + # + # the init function + prnt('#ifdef __GNUC__') + prnt('# pragma GCC visibility push(default) /* for -fvisibility= */') + prnt('#endif') + prnt() + prnt('#ifdef PYPY_VERSION') + prnt('PyMODINIT_FUNC') + prnt('_cffi_pypyinit_%s(const void *p[])' % (base_module_name,)) + prnt('{') + if self._num_externpy: + prnt(' if (((intptr_t)p[0]) >= 0x0A03) {') + prnt(' _cffi_call_python_org = ' + '(void(*)(struct _cffi_externpy_s *, char *))p[1];') + prnt(' }') + prnt(' p[0] = (const void *)0x%x;' % self._version) + prnt(' p[1] = &_cffi_type_context;') + prnt('#if PY_MAJOR_VERSION >= 3') + prnt(' return NULL;') + prnt('#endif') + prnt('}') + # on Windows, distutils insists on putting init_cffi_xyz in + # 'export_symbols', so instead of fighting it, just give up and + # give it one + prnt('# ifdef _MSC_VER') + prnt(' PyMODINIT_FUNC') + prnt('# if PY_MAJOR_VERSION >= 3') + prnt(' PyInit_%s(void) { return NULL; }' % (base_module_name,)) + prnt('# else') + prnt(' init%s(void) { }' % (base_module_name,)) + prnt('# endif') + prnt('# endif') + prnt('#elif PY_MAJOR_VERSION >= 3') + prnt('PyMODINIT_FUNC') + prnt('PyInit_%s(void)' % (base_module_name,)) + prnt('{') + prnt(' return _cffi_init("%s", 0x%x, &_cffi_type_context);' % ( + self.module_name, self._version)) + prnt('}') + prnt('#else') + prnt('PyMODINIT_FUNC') + prnt('init%s(void)' % (base_module_name,)) + prnt('{') + prnt(' _cffi_init("%s", 0x%x, &_cffi_type_context);' % ( + self.module_name, self._version)) + prnt('}') + prnt('#endif') + prnt() + prnt('#ifdef __GNUC__') + prnt('# pragma GCC visibility pop') + prnt('#endif') + self._version = None + + def _to_py(self, x): + if isinstance(x, str): + return "b'%s'" % (x,) + if isinstance(x, (list, tuple)): + rep = [self._to_py(item) for item in x] + if len(rep) == 1: + rep.append('') + return "(%s)" % (','.join(rep),) + return x.as_python_expr() # Py2: unicode unexpected; Py3: bytes unexp. + + def write_py_source_to_f(self, f): + self._f = f + prnt = self._prnt + # + # header + prnt("# auto-generated file") + prnt("import _cffi_backend") + # + # the 'import' of the included ffis + num_includes = len(self.ffi._included_ffis or ()) + for i in range(num_includes): + ffi_to_include = self.ffi._included_ffis[i] + try: + included_module_name, included_source = ( + ffi_to_include._assigned_source[:2]) + except AttributeError: + raise VerificationError( + "ffi object %r includes %r, but the latter has not " + "been prepared with set_source()" % ( + self.ffi, ffi_to_include,)) + if included_source is not None: + raise VerificationError( + "not implemented yet: ffi.include() of a C-based " + "ffi inside a Python-based ffi") + prnt('from %s import ffi as _ffi%d' % (included_module_name, i)) + prnt() + prnt("ffi = _cffi_backend.FFI('%s'," % (self.module_name,)) + prnt(" _version = 0x%x," % (self._version,)) + self._version = None + # + # the '_types' keyword argument + self.cffi_types = tuple(self.cffi_types) # don't change any more + types_lst = [op.as_python_bytes() for op in self.cffi_types] + prnt(' _types = %s,' % (self._to_py(''.join(types_lst)),)) + typeindex2type = dict([(i, tp) for (tp, i) in self._typesdict.items()]) + # + # the keyword arguments from ALL_STEPS + for step_name in self.ALL_STEPS: + lst = self._lsts[step_name] + if len(lst) > 0 and step_name != "field": + prnt(' _%ss = %s,' % (step_name, self._to_py(lst))) + # + # the '_includes' keyword argument + if num_includes > 0: + prnt(' _includes = (%s,),' % ( + ', '.join(['_ffi%d' % i for i in range(num_includes)]),)) + # + # the footer + prnt(')') + + # ---------- + + def _gettypenum(self, type): + # a KeyError here is a bug. please report it! :-) + return self._typesdict[type] + + def _convert_funcarg_to_c(self, tp, fromvar, tovar, errcode): + extraarg = '' + if isinstance(tp, model.BasePrimitiveType) and not tp.is_complex_type(): + if tp.is_integer_type() and tp.name != '_Bool': + converter = '_cffi_to_c_int' + extraarg = ', %s' % tp.name + elif isinstance(tp, model.UnknownFloatType): + # don't check with is_float_type(): it may be a 'long + # double' here, and _cffi_to_c_double would loose precision + converter = '(%s)_cffi_to_c_double' % (tp.get_c_name(''),) + else: + cname = tp.get_c_name('') + converter = '(%s)_cffi_to_c_%s' % (cname, + tp.name.replace(' ', '_')) + if cname in ('char16_t', 'char32_t'): + self.needs_version(VERSION_CHAR16CHAR32) + errvalue = '-1' + # + elif isinstance(tp, model.PointerType): + self._convert_funcarg_to_c_ptr_or_array(tp, fromvar, + tovar, errcode) + return + # + elif (isinstance(tp, model.StructOrUnionOrEnum) or + isinstance(tp, model.BasePrimitiveType)): + # a struct (not a struct pointer) as a function argument; + # or, a complex (the same code works) + self._prnt(' if (_cffi_to_c((char *)&%s, _cffi_type(%d), %s) < 0)' + % (tovar, self._gettypenum(tp), fromvar)) + self._prnt(' %s;' % errcode) + return + # + elif isinstance(tp, model.FunctionPtrType): + converter = '(%s)_cffi_to_c_pointer' % tp.get_c_name('') + extraarg = ', _cffi_type(%d)' % self._gettypenum(tp) + errvalue = 'NULL' + # + else: + raise NotImplementedError(tp) + # + self._prnt(' %s = %s(%s%s);' % (tovar, converter, fromvar, extraarg)) + self._prnt(' if (%s == (%s)%s && PyErr_Occurred())' % ( + tovar, tp.get_c_name(''), errvalue)) + self._prnt(' %s;' % errcode) + + def _extra_local_variables(self, tp, localvars, freelines): + if isinstance(tp, model.PointerType): + localvars.add('Py_ssize_t datasize') + localvars.add('struct _cffi_freeme_s *large_args_free = NULL') + freelines.add('if (large_args_free != NULL)' + ' _cffi_free_array_arguments(large_args_free);') + + def _convert_funcarg_to_c_ptr_or_array(self, tp, fromvar, tovar, errcode): + self._prnt(' datasize = _cffi_prepare_pointer_call_argument(') + self._prnt(' _cffi_type(%d), %s, (char **)&%s);' % ( + self._gettypenum(tp), fromvar, tovar)) + self._prnt(' if (datasize != 0) {') + self._prnt(' %s = ((size_t)datasize) <= 640 ? ' + '(%s)alloca((size_t)datasize) : NULL;' % ( + tovar, tp.get_c_name(''))) + self._prnt(' if (_cffi_convert_array_argument(_cffi_type(%d), %s, ' + '(char **)&%s,' % (self._gettypenum(tp), fromvar, tovar)) + self._prnt(' datasize, &large_args_free) < 0)') + self._prnt(' %s;' % errcode) + self._prnt(' }') + + def _convert_expr_from_c(self, tp, var, context): + if isinstance(tp, model.BasePrimitiveType): + if tp.is_integer_type() and tp.name != '_Bool': + return '_cffi_from_c_int(%s, %s)' % (var, tp.name) + elif isinstance(tp, model.UnknownFloatType): + return '_cffi_from_c_double(%s)' % (var,) + elif tp.name != 'long double' and not tp.is_complex_type(): + cname = tp.name.replace(' ', '_') + if cname in ('char16_t', 'char32_t'): + self.needs_version(VERSION_CHAR16CHAR32) + return '_cffi_from_c_%s(%s)' % (cname, var) + else: + return '_cffi_from_c_deref((char *)&%s, _cffi_type(%d))' % ( + var, self._gettypenum(tp)) + elif isinstance(tp, (model.PointerType, model.FunctionPtrType)): + return '_cffi_from_c_pointer((char *)%s, _cffi_type(%d))' % ( + var, self._gettypenum(tp)) + elif isinstance(tp, model.ArrayType): + return '_cffi_from_c_pointer((char *)%s, _cffi_type(%d))' % ( + var, self._gettypenum(model.PointerType(tp.item))) + elif isinstance(tp, model.StructOrUnion): + if tp.fldnames is None: + raise TypeError("'%s' is used as %s, but is opaque" % ( + tp._get_c_name(), context)) + return '_cffi_from_c_struct((char *)&%s, _cffi_type(%d))' % ( + var, self._gettypenum(tp)) + elif isinstance(tp, model.EnumType): + return '_cffi_from_c_deref((char *)&%s, _cffi_type(%d))' % ( + var, self._gettypenum(tp)) + else: + raise NotImplementedError(tp) + + # ---------- + # typedefs + + def _typedef_type(self, tp, name): + return self._global_type(tp, "(*(%s *)0)" % (name,)) + + def _generate_cpy_typedef_collecttype(self, tp, name): + self._do_collect_type(self._typedef_type(tp, name)) + + def _generate_cpy_typedef_decl(self, tp, name): + pass + + def _typedef_ctx(self, tp, name): + type_index = self._typesdict[tp] + self._lsts["typename"].append(TypenameExpr(name, type_index)) + + def _generate_cpy_typedef_ctx(self, tp, name): + tp = self._typedef_type(tp, name) + self._typedef_ctx(tp, name) + if getattr(tp, "origin", None) == "unknown_type": + self._struct_ctx(tp, tp.name, approxname=None) + elif isinstance(tp, model.NamedPointerType): + self._struct_ctx(tp.totype, tp.totype.name, approxname=tp.name, + named_ptr=tp) + + # ---------- + # function declarations + + def _generate_cpy_function_collecttype(self, tp, name): + self._do_collect_type(tp.as_raw_function()) + if tp.ellipsis and not self.target_is_python: + self._do_collect_type(tp) + + def _generate_cpy_function_decl(self, tp, name): + assert not self.target_is_python + assert isinstance(tp, model.FunctionPtrType) + if tp.ellipsis: + # cannot support vararg functions better than this: check for its + # exact type (including the fixed arguments), and build it as a + # constant function pointer (no CPython wrapper) + self._generate_cpy_constant_decl(tp, name) + return + prnt = self._prnt + numargs = len(tp.args) + if numargs == 0: + argname = 'noarg' + elif numargs == 1: + argname = 'arg0' + else: + argname = 'args' + # + # ------------------------------ + # the 'd' version of the function, only for addressof(lib, 'func') + arguments = [] + call_arguments = [] + context = 'argument of %s' % name + for i, type in enumerate(tp.args): + arguments.append(type.get_c_name(' x%d' % i, context)) + call_arguments.append('x%d' % i) + repr_arguments = ', '.join(arguments) + repr_arguments = repr_arguments or 'void' + if tp.abi: + abi = tp.abi + ' ' + else: + abi = '' + name_and_arguments = '%s_cffi_d_%s(%s)' % (abi, name, repr_arguments) + prnt('static %s' % (tp.result.get_c_name(name_and_arguments),)) + prnt('{') + call_arguments = ', '.join(call_arguments) + result_code = 'return ' + if isinstance(tp.result, model.VoidType): + result_code = '' + prnt(' %s%s(%s);' % (result_code, name, call_arguments)) + prnt('}') + # + prnt('#ifndef PYPY_VERSION') # ------------------------------ + # + prnt('static PyObject *') + prnt('_cffi_f_%s(PyObject *self, PyObject *%s)' % (name, argname)) + prnt('{') + # + context = 'argument of %s' % name + for i, type in enumerate(tp.args): + arg = type.get_c_name(' x%d' % i, context) + prnt(' %s;' % arg) + # + localvars = set() + freelines = set() + for type in tp.args: + self._extra_local_variables(type, localvars, freelines) + for decl in sorted(localvars): + prnt(' %s;' % (decl,)) + # + if not isinstance(tp.result, model.VoidType): + result_code = 'result = ' + context = 'result of %s' % name + result_decl = ' %s;' % tp.result.get_c_name(' result', context) + prnt(result_decl) + prnt(' PyObject *pyresult;') + else: + result_decl = None + result_code = '' + # + if len(tp.args) > 1: + rng = range(len(tp.args)) + for i in rng: + prnt(' PyObject *arg%d;' % i) + prnt() + prnt(' if (!PyArg_UnpackTuple(args, "%s", %d, %d, %s))' % ( + name, len(rng), len(rng), + ', '.join(['&arg%d' % i for i in rng]))) + prnt(' return NULL;') + prnt() + # + for i, type in enumerate(tp.args): + self._convert_funcarg_to_c(type, 'arg%d' % i, 'x%d' % i, + 'return NULL') + prnt() + # + prnt(' Py_BEGIN_ALLOW_THREADS') + prnt(' _cffi_restore_errno();') + call_arguments = ['x%d' % i for i in range(len(tp.args))] + call_arguments = ', '.join(call_arguments) + prnt(' { %s%s(%s); }' % (result_code, name, call_arguments)) + prnt(' _cffi_save_errno();') + prnt(' Py_END_ALLOW_THREADS') + prnt() + # + prnt(' (void)self; /* unused */') + if numargs == 0: + prnt(' (void)noarg; /* unused */') + if result_code: + prnt(' pyresult = %s;' % + self._convert_expr_from_c(tp.result, 'result', 'result type')) + for freeline in freelines: + prnt(' ' + freeline) + prnt(' return pyresult;') + else: + for freeline in freelines: + prnt(' ' + freeline) + prnt(' Py_INCREF(Py_None);') + prnt(' return Py_None;') + prnt('}') + # + prnt('#else') # ------------------------------ + # + # the PyPy version: need to replace struct/union arguments with + # pointers, and if the result is a struct/union, insert a first + # arg that is a pointer to the result. We also do that for + # complex args and return type. + def need_indirection(type): + return (isinstance(type, model.StructOrUnion) or + (isinstance(type, model.PrimitiveType) and + type.is_complex_type())) + difference = False + arguments = [] + call_arguments = [] + context = 'argument of %s' % name + for i, type in enumerate(tp.args): + indirection = '' + if need_indirection(type): + indirection = '*' + difference = True + arg = type.get_c_name(' %sx%d' % (indirection, i), context) + arguments.append(arg) + call_arguments.append('%sx%d' % (indirection, i)) + tp_result = tp.result + if need_indirection(tp_result): + context = 'result of %s' % name + arg = tp_result.get_c_name(' *result', context) + arguments.insert(0, arg) + tp_result = model.void_type + result_decl = None + result_code = '*result = ' + difference = True + if difference: + repr_arguments = ', '.join(arguments) + repr_arguments = repr_arguments or 'void' + name_and_arguments = '%s_cffi_f_%s(%s)' % (abi, name, + repr_arguments) + prnt('static %s' % (tp_result.get_c_name(name_and_arguments),)) + prnt('{') + if result_decl: + prnt(result_decl) + call_arguments = ', '.join(call_arguments) + prnt(' { %s%s(%s); }' % (result_code, name, call_arguments)) + if result_decl: + prnt(' return result;') + prnt('}') + else: + prnt('# define _cffi_f_%s _cffi_d_%s' % (name, name)) + # + prnt('#endif') # ------------------------------ + prnt() + + def _generate_cpy_function_ctx(self, tp, name): + if tp.ellipsis and not self.target_is_python: + self._generate_cpy_constant_ctx(tp, name) + return + type_index = self._typesdict[tp.as_raw_function()] + numargs = len(tp.args) + if self.target_is_python: + meth_kind = OP_DLOPEN_FUNC + elif numargs == 0: + meth_kind = OP_CPYTHON_BLTN_N # 'METH_NOARGS' + elif numargs == 1: + meth_kind = OP_CPYTHON_BLTN_O # 'METH_O' + else: + meth_kind = OP_CPYTHON_BLTN_V # 'METH_VARARGS' + self._lsts["global"].append( + GlobalExpr(name, '_cffi_f_%s' % name, + CffiOp(meth_kind, type_index), + size='_cffi_d_%s' % name)) + + # ---------- + # named structs or unions + + def _field_type(self, tp_struct, field_name, tp_field): + if isinstance(tp_field, model.ArrayType): + actual_length = tp_field.length + if actual_length == '...': + ptr_struct_name = tp_struct.get_c_name('*') + actual_length = '_cffi_array_len(((%s)0)->%s)' % ( + ptr_struct_name, field_name) + tp_item = self._field_type(tp_struct, '%s[0]' % field_name, + tp_field.item) + tp_field = model.ArrayType(tp_item, actual_length) + return tp_field + + def _struct_collecttype(self, tp): + self._do_collect_type(tp) + if self.target_is_python: + # also requires nested anon struct/unions in ABI mode, recursively + for fldtype in tp.anonymous_struct_fields(): + self._struct_collecttype(fldtype) + + def _struct_decl(self, tp, cname, approxname): + if tp.fldtypes is None: + return + prnt = self._prnt + checkfuncname = '_cffi_checkfld_%s' % (approxname,) + prnt('_CFFI_UNUSED_FN') + prnt('static void %s(%s *p)' % (checkfuncname, cname)) + prnt('{') + prnt(' /* only to generate compile-time warnings or errors */') + prnt(' (void)p;') + for fname, ftype, fbitsize, fqual in self._enum_fields(tp): + try: + if ftype.is_integer_type() or fbitsize >= 0: + # accept all integers, but complain on float or double + if fname != '': + prnt(" (void)((p->%s) | 0); /* check that '%s.%s' is " + "an integer */" % (fname, cname, fname)) + continue + # only accept exactly the type declared, except that '[]' + # is interpreted as a '*' and so will match any array length. + # (It would also match '*', but that's harder to detect...) + while (isinstance(ftype, model.ArrayType) + and (ftype.length is None or ftype.length == '...')): + ftype = ftype.item + fname = fname + '[0]' + prnt(' { %s = &p->%s; (void)tmp; }' % ( + ftype.get_c_name('*tmp', 'field %r'%fname, quals=fqual), + fname)) + except VerificationError as e: + prnt(' /* %s */' % str(e)) # cannot verify it, ignore + prnt('}') + prnt('struct _cffi_align_%s { char x; %s y; };' % (approxname, cname)) + prnt() + + def _struct_ctx(self, tp, cname, approxname, named_ptr=None): + type_index = self._typesdict[tp] + reason_for_not_expanding = None + flags = [] + if isinstance(tp, model.UnionType): + flags.append("_CFFI_F_UNION") + if tp.fldtypes is None: + flags.append("_CFFI_F_OPAQUE") + reason_for_not_expanding = "opaque" + if (tp not in self.ffi._parser._included_declarations and + (named_ptr is None or + named_ptr not in self.ffi._parser._included_declarations)): + if tp.fldtypes is None: + pass # opaque + elif tp.partial or any(tp.anonymous_struct_fields()): + pass # field layout obtained silently from the C compiler + else: + flags.append("_CFFI_F_CHECK_FIELDS") + if tp.packed: + if tp.packed > 1: + raise NotImplementedError( + "%r is declared with 'pack=%r'; only 0 or 1 are " + "supported in API mode (try to use \"...;\", which " + "does not require a 'pack' declaration)" % + (tp, tp.packed)) + flags.append("_CFFI_F_PACKED") + else: + flags.append("_CFFI_F_EXTERNAL") + reason_for_not_expanding = "external" + flags = '|'.join(flags) or '0' + c_fields = [] + if reason_for_not_expanding is None: + enumfields = list(self._enum_fields(tp)) + for fldname, fldtype, fbitsize, fqual in enumfields: + fldtype = self._field_type(tp, fldname, fldtype) + self._check_not_opaque(fldtype, + "field '%s.%s'" % (tp.name, fldname)) + # cname is None for _add_missing_struct_unions() only + op = OP_NOOP + if fbitsize >= 0: + op = OP_BITFIELD + size = '%d /* bits */' % fbitsize + elif cname is None or ( + isinstance(fldtype, model.ArrayType) and + fldtype.length is None): + size = '(size_t)-1' + else: + size = 'sizeof(((%s)0)->%s)' % ( + tp.get_c_name('*') if named_ptr is None + else named_ptr.name, + fldname) + if cname is None or fbitsize >= 0: + offset = '(size_t)-1' + elif named_ptr is not None: + offset = '((char *)&((%s)0)->%s) - (char *)0' % ( + named_ptr.name, fldname) + else: + offset = 'offsetof(%s, %s)' % (tp.get_c_name(''), fldname) + c_fields.append( + FieldExpr(fldname, offset, size, fbitsize, + CffiOp(op, self._typesdict[fldtype]))) + first_field_index = len(self._lsts["field"]) + self._lsts["field"].extend(c_fields) + # + if cname is None: # unknown name, for _add_missing_struct_unions + size = '(size_t)-2' + align = -2 + comment = "unnamed" + else: + if named_ptr is not None: + size = 'sizeof(*(%s)0)' % (named_ptr.name,) + align = '-1 /* unknown alignment */' + else: + size = 'sizeof(%s)' % (cname,) + align = 'offsetof(struct _cffi_align_%s, y)' % (approxname,) + comment = None + else: + size = '(size_t)-1' + align = -1 + first_field_index = -1 + comment = reason_for_not_expanding + self._lsts["struct_union"].append( + StructUnionExpr(tp.name, type_index, flags, size, align, comment, + first_field_index, c_fields)) + self._seen_struct_unions.add(tp) + + def _check_not_opaque(self, tp, location): + while isinstance(tp, model.ArrayType): + tp = tp.item + if isinstance(tp, model.StructOrUnion) and tp.fldtypes is None: + raise TypeError( + "%s is of an opaque type (not declared in cdef())" % location) + + def _add_missing_struct_unions(self): + # not very nice, but some struct declarations might be missing + # because they don't have any known C name. Check that they are + # not partial (we can't complete or verify them!) and emit them + # anonymously. + lst = list(self._struct_unions.items()) + lst.sort(key=lambda tp_order: tp_order[1]) + for tp, order in lst: + if tp not in self._seen_struct_unions: + if tp.partial: + raise NotImplementedError("internal inconsistency: %r is " + "partial but was not seen at " + "this point" % (tp,)) + if tp.name.startswith('$') and tp.name[1:].isdigit(): + approxname = tp.name[1:] + elif tp.name == '_IO_FILE' and tp.forcename == 'FILE': + approxname = 'FILE' + self._typedef_ctx(tp, 'FILE') + else: + raise NotImplementedError("internal inconsistency: %r" % + (tp,)) + self._struct_ctx(tp, None, approxname) + + def _generate_cpy_struct_collecttype(self, tp, name): + self._struct_collecttype(tp) + _generate_cpy_union_collecttype = _generate_cpy_struct_collecttype + + def _struct_names(self, tp): + cname = tp.get_c_name('') + if ' ' in cname: + return cname, cname.replace(' ', '_') + else: + return cname, '_' + cname + + def _generate_cpy_struct_decl(self, tp, name): + self._struct_decl(tp, *self._struct_names(tp)) + _generate_cpy_union_decl = _generate_cpy_struct_decl + + def _generate_cpy_struct_ctx(self, tp, name): + self._struct_ctx(tp, *self._struct_names(tp)) + _generate_cpy_union_ctx = _generate_cpy_struct_ctx + + # ---------- + # 'anonymous' declarations. These are produced for anonymous structs + # or unions; the 'name' is obtained by a typedef. + + def _generate_cpy_anonymous_collecttype(self, tp, name): + if isinstance(tp, model.EnumType): + self._generate_cpy_enum_collecttype(tp, name) + else: + self._struct_collecttype(tp) + + def _generate_cpy_anonymous_decl(self, tp, name): + if isinstance(tp, model.EnumType): + self._generate_cpy_enum_decl(tp) + else: + self._struct_decl(tp, name, 'typedef_' + name) + + def _generate_cpy_anonymous_ctx(self, tp, name): + if isinstance(tp, model.EnumType): + self._enum_ctx(tp, name) + else: + self._struct_ctx(tp, name, 'typedef_' + name) + + # ---------- + # constants, declared with "static const ..." + + def _generate_cpy_const(self, is_int, name, tp=None, category='const', + check_value=None): + if (category, name) in self._seen_constants: + raise VerificationError( + "duplicate declaration of %s '%s'" % (category, name)) + self._seen_constants.add((category, name)) + # + prnt = self._prnt + funcname = '_cffi_%s_%s' % (category, name) + if is_int: + prnt('static int %s(unsigned long long *o)' % funcname) + prnt('{') + prnt(' int n = (%s) <= 0;' % (name,)) + prnt(' *o = (unsigned long long)((%s) | 0);' + ' /* check that %s is an integer */' % (name, name)) + if check_value is not None: + if check_value > 0: + check_value = '%dU' % (check_value,) + prnt(' if (!_cffi_check_int(*o, n, %s))' % (check_value,)) + prnt(' n |= 2;') + prnt(' return n;') + prnt('}') + else: + assert check_value is None + prnt('static void %s(char *o)' % funcname) + prnt('{') + prnt(' *(%s)o = %s;' % (tp.get_c_name('*'), name)) + prnt('}') + prnt() + + def _generate_cpy_constant_collecttype(self, tp, name): + is_int = tp.is_integer_type() + if not is_int or self.target_is_python: + self._do_collect_type(tp) + + def _generate_cpy_constant_decl(self, tp, name): + is_int = tp.is_integer_type() + self._generate_cpy_const(is_int, name, tp) + + def _generate_cpy_constant_ctx(self, tp, name): + if not self.target_is_python and tp.is_integer_type(): + type_op = CffiOp(OP_CONSTANT_INT, -1) + else: + if self.target_is_python: + const_kind = OP_DLOPEN_CONST + else: + const_kind = OP_CONSTANT + type_index = self._typesdict[tp] + type_op = CffiOp(const_kind, type_index) + self._lsts["global"].append( + GlobalExpr(name, '_cffi_const_%s' % name, type_op)) + + # ---------- + # enums + + def _generate_cpy_enum_collecttype(self, tp, name): + self._do_collect_type(tp) + + def _generate_cpy_enum_decl(self, tp, name=None): + for enumerator in tp.enumerators: + self._generate_cpy_const(True, enumerator) + + def _enum_ctx(self, tp, cname): + type_index = self._typesdict[tp] + type_op = CffiOp(OP_ENUM, -1) + if self.target_is_python: + tp.check_not_partial() + for enumerator, enumvalue in zip(tp.enumerators, tp.enumvalues): + self._lsts["global"].append( + GlobalExpr(enumerator, '_cffi_const_%s' % enumerator, type_op, + check_value=enumvalue)) + # + if cname is not None and '$' not in cname and not self.target_is_python: + size = "sizeof(%s)" % cname + signed = "((%s)-1) <= 0" % cname + else: + basetp = tp.build_baseinttype(self.ffi, []) + size = self.ffi.sizeof(basetp) + signed = int(int(self.ffi.cast(basetp, -1)) < 0) + allenums = ",".join(tp.enumerators) + self._lsts["enum"].append( + EnumExpr(tp.name, type_index, size, signed, allenums)) + + def _generate_cpy_enum_ctx(self, tp, name): + self._enum_ctx(tp, tp._get_c_name()) + + # ---------- + # macros: for now only for integers + + def _generate_cpy_macro_collecttype(self, tp, name): + pass + + def _generate_cpy_macro_decl(self, tp, name): + if tp == '...': + check_value = None + else: + check_value = tp # an integer + self._generate_cpy_const(True, name, check_value=check_value) + + def _generate_cpy_macro_ctx(self, tp, name): + if tp == '...': + if self.target_is_python: + raise VerificationError( + "cannot use the syntax '...' in '#define %s ...' when " + "using the ABI mode" % (name,)) + check_value = None + else: + check_value = tp # an integer + type_op = CffiOp(OP_CONSTANT_INT, -1) + self._lsts["global"].append( + GlobalExpr(name, '_cffi_const_%s' % name, type_op, + check_value=check_value)) + + # ---------- + # global variables + + def _global_type(self, tp, global_name): + if isinstance(tp, model.ArrayType): + actual_length = tp.length + if actual_length == '...': + actual_length = '_cffi_array_len(%s)' % (global_name,) + tp_item = self._global_type(tp.item, '%s[0]' % global_name) + tp = model.ArrayType(tp_item, actual_length) + return tp + + def _generate_cpy_variable_collecttype(self, tp, name): + self._do_collect_type(self._global_type(tp, name)) + + def _generate_cpy_variable_decl(self, tp, name): + prnt = self._prnt + tp = self._global_type(tp, name) + if isinstance(tp, model.ArrayType) and tp.length is None: + tp = tp.item + ampersand = '' + else: + ampersand = '&' + # This code assumes that casts from "tp *" to "void *" is a + # no-op, i.e. a function that returns a "tp *" can be called + # as if it returned a "void *". This should be generally true + # on any modern machine. The only exception to that rule (on + # uncommon architectures, and as far as I can tell) might be + # if 'tp' were a function type, but that is not possible here. + # (If 'tp' is a function _pointer_ type, then casts from "fn_t + # **" to "void *" are again no-ops, as far as I can tell.) + decl = '*_cffi_var_%s(void)' % (name,) + prnt('static ' + tp.get_c_name(decl, quals=self._current_quals)) + prnt('{') + prnt(' return %s(%s);' % (ampersand, name)) + prnt('}') + prnt() + + def _generate_cpy_variable_ctx(self, tp, name): + tp = self._global_type(tp, name) + type_index = self._typesdict[tp] + if self.target_is_python: + op = OP_GLOBAL_VAR + else: + op = OP_GLOBAL_VAR_F + self._lsts["global"].append( + GlobalExpr(name, '_cffi_var_%s' % name, CffiOp(op, type_index))) + + # ---------- + # extern "Python" + + def _generate_cpy_extern_python_collecttype(self, tp, name): + assert isinstance(tp, model.FunctionPtrType) + self._do_collect_type(tp) + _generate_cpy_dllexport_python_collecttype = \ + _generate_cpy_extern_python_plus_c_collecttype = \ + _generate_cpy_extern_python_collecttype + + def _extern_python_decl(self, tp, name, tag_and_space): + prnt = self._prnt + if isinstance(tp.result, model.VoidType): + size_of_result = '0' + else: + context = 'result of %s' % name + size_of_result = '(int)sizeof(%s)' % ( + tp.result.get_c_name('', context),) + prnt('static struct _cffi_externpy_s _cffi_externpy__%s =' % name) + prnt(' { "%s.%s", %s, 0, 0 };' % ( + self.module_name, name, size_of_result)) + prnt() + # + arguments = [] + context = 'argument of %s' % name + for i, type in enumerate(tp.args): + arg = type.get_c_name(' a%d' % i, context) + arguments.append(arg) + # + repr_arguments = ', '.join(arguments) + repr_arguments = repr_arguments or 'void' + name_and_arguments = '%s(%s)' % (name, repr_arguments) + if tp.abi == "__stdcall": + name_and_arguments = '_cffi_stdcall ' + name_and_arguments + # + def may_need_128_bits(tp): + return (isinstance(tp, model.PrimitiveType) and + tp.name == 'long double') + # + size_of_a = max(len(tp.args)*8, 8) + if may_need_128_bits(tp.result): + size_of_a = max(size_of_a, 16) + if isinstance(tp.result, model.StructOrUnion): + size_of_a = 'sizeof(%s) > %d ? sizeof(%s) : %d' % ( + tp.result.get_c_name(''), size_of_a, + tp.result.get_c_name(''), size_of_a) + prnt('%s%s' % (tag_and_space, tp.result.get_c_name(name_and_arguments))) + prnt('{') + prnt(' char a[%s];' % size_of_a) + prnt(' char *p = a;') + for i, type in enumerate(tp.args): + arg = 'a%d' % i + if (isinstance(type, model.StructOrUnion) or + may_need_128_bits(type)): + arg = '&' + arg + type = model.PointerType(type) + prnt(' *(%s)(p + %d) = %s;' % (type.get_c_name('*'), i*8, arg)) + prnt(' _cffi_call_python(&_cffi_externpy__%s, p);' % name) + if not isinstance(tp.result, model.VoidType): + prnt(' return *(%s)p;' % (tp.result.get_c_name('*'),)) + prnt('}') + prnt() + self._num_externpy += 1 + + def _generate_cpy_extern_python_decl(self, tp, name): + self._extern_python_decl(tp, name, 'static ') + + def _generate_cpy_dllexport_python_decl(self, tp, name): + self._extern_python_decl(tp, name, 'CFFI_DLLEXPORT ') + + def _generate_cpy_extern_python_plus_c_decl(self, tp, name): + self._extern_python_decl(tp, name, '') + + def _generate_cpy_extern_python_ctx(self, tp, name): + if self.target_is_python: + raise VerificationError( + "cannot use 'extern \"Python\"' in the ABI mode") + if tp.ellipsis: + raise NotImplementedError("a vararg function is extern \"Python\"") + type_index = self._typesdict[tp] + type_op = CffiOp(OP_EXTERN_PYTHON, type_index) + self._lsts["global"].append( + GlobalExpr(name, '&_cffi_externpy__%s' % name, type_op, name)) + + _generate_cpy_dllexport_python_ctx = \ + _generate_cpy_extern_python_plus_c_ctx = \ + _generate_cpy_extern_python_ctx + + def _print_string_literal_in_array(self, s): + prnt = self._prnt + prnt('// # NB. this is not a string because of a size limit in MSVC') + if not isinstance(s, bytes): # unicode + s = s.encode('utf-8') # -> bytes + else: + s.decode('utf-8') # got bytes, check for valid utf-8 + try: + s.decode('ascii') + except UnicodeDecodeError: + s = b'# -*- encoding: utf8 -*-\n' + s + for line in s.splitlines(True): + comment = line + if type('//') is bytes: # python2 + line = map(ord, line) # make a list of integers + else: # python3 + # type(line) is bytes, which enumerates like a list of integers + comment = ascii(comment)[1:-1] + prnt(('// ' + comment).rstrip()) + printed_line = '' + for c in line: + if len(printed_line) >= 76: + prnt(printed_line) + printed_line = '' + printed_line += '%d,' % (c,) + prnt(printed_line) + + # ---------- + # emitting the opcodes for individual types + + def _emit_bytecode_VoidType(self, tp, index): + self.cffi_types[index] = CffiOp(OP_PRIMITIVE, PRIM_VOID) + + def _emit_bytecode_PrimitiveType(self, tp, index): + prim_index = PRIMITIVE_TO_INDEX[tp.name] + self.cffi_types[index] = CffiOp(OP_PRIMITIVE, prim_index) + + def _emit_bytecode_UnknownIntegerType(self, tp, index): + s = ('_cffi_prim_int(sizeof(%s), (\n' + ' ((%s)-1) | 0 /* check that %s is an integer type */\n' + ' ) <= 0)' % (tp.name, tp.name, tp.name)) + self.cffi_types[index] = CffiOp(OP_PRIMITIVE, s) + + def _emit_bytecode_UnknownFloatType(self, tp, index): + s = ('_cffi_prim_float(sizeof(%s) *\n' + ' (((%s)1) / 2) * 2 /* integer => 0, float => 1 */\n' + ' )' % (tp.name, tp.name)) + self.cffi_types[index] = CffiOp(OP_PRIMITIVE, s) + + def _emit_bytecode_RawFunctionType(self, tp, index): + self.cffi_types[index] = CffiOp(OP_FUNCTION, self._typesdict[tp.result]) + index += 1 + for tp1 in tp.args: + realindex = self._typesdict[tp1] + if index != realindex: + if isinstance(tp1, model.PrimitiveType): + self._emit_bytecode_PrimitiveType(tp1, index) + else: + self.cffi_types[index] = CffiOp(OP_NOOP, realindex) + index += 1 + flags = int(tp.ellipsis) + if tp.abi is not None: + if tp.abi == '__stdcall': + flags |= 2 + else: + raise NotImplementedError("abi=%r" % (tp.abi,)) + self.cffi_types[index] = CffiOp(OP_FUNCTION_END, flags) + + def _emit_bytecode_PointerType(self, tp, index): + self.cffi_types[index] = CffiOp(OP_POINTER, self._typesdict[tp.totype]) + + _emit_bytecode_ConstPointerType = _emit_bytecode_PointerType + _emit_bytecode_NamedPointerType = _emit_bytecode_PointerType + + def _emit_bytecode_FunctionPtrType(self, tp, index): + raw = tp.as_raw_function() + self.cffi_types[index] = CffiOp(OP_POINTER, self._typesdict[raw]) + + def _emit_bytecode_ArrayType(self, tp, index): + item_index = self._typesdict[tp.item] + if tp.length is None: + self.cffi_types[index] = CffiOp(OP_OPEN_ARRAY, item_index) + elif tp.length == '...': + raise VerificationError( + "type %s badly placed: the '...' array length can only be " + "used on global arrays or on fields of structures" % ( + str(tp).replace('/*...*/', '...'),)) + else: + assert self.cffi_types[index + 1] == 'LEN' + self.cffi_types[index] = CffiOp(OP_ARRAY, item_index) + self.cffi_types[index + 1] = CffiOp(None, str(tp.length)) + + def _emit_bytecode_StructType(self, tp, index): + struct_index = self._struct_unions[tp] + self.cffi_types[index] = CffiOp(OP_STRUCT_UNION, struct_index) + _emit_bytecode_UnionType = _emit_bytecode_StructType + + def _emit_bytecode_EnumType(self, tp, index): + enum_index = self._enums[tp] + self.cffi_types[index] = CffiOp(OP_ENUM, enum_index) + + +if sys.version_info >= (3,): + NativeIO = io.StringIO +else: + class NativeIO(io.BytesIO): + def write(self, s): + if isinstance(s, unicode): + s = s.encode('ascii') + super(NativeIO, self).write(s) + +def _make_c_or_py_source(ffi, module_name, preamble, target_file, verbose): + if verbose: + print("generating %s" % (target_file,)) + recompiler = Recompiler(ffi, module_name, + target_is_python=(preamble is None)) + recompiler.collect_type_table() + recompiler.collect_step_tables() + f = NativeIO() + recompiler.write_source_to_f(f, preamble) + output = f.getvalue() + try: + with open(target_file, 'r') as f1: + if f1.read(len(output) + 1) != output: + raise IOError + if verbose: + print("(already up-to-date)") + return False # already up-to-date + except IOError: + tmp_file = '%s.~%d' % (target_file, os.getpid()) + with open(tmp_file, 'w') as f1: + f1.write(output) + try: + os.rename(tmp_file, target_file) + except OSError: + os.unlink(target_file) + os.rename(tmp_file, target_file) + return True + +def make_c_source(ffi, module_name, preamble, target_c_file, verbose=False): + assert preamble is not None + return _make_c_or_py_source(ffi, module_name, preamble, target_c_file, + verbose) + +def make_py_source(ffi, module_name, target_py_file, verbose=False): + return _make_c_or_py_source(ffi, module_name, None, target_py_file, + verbose) + +def _modname_to_file(outputdir, modname, extension): + parts = modname.split('.') + try: + os.makedirs(os.path.join(outputdir, *parts[:-1])) + except OSError: + pass + parts[-1] += extension + return os.path.join(outputdir, *parts), parts + + +# Aaargh. Distutils is not tested at all for the purpose of compiling +# DLLs that are not extension modules. Here are some hacks to work +# around that, in the _patch_for_*() functions... + +def _patch_meth(patchlist, cls, name, new_meth): + old = getattr(cls, name) + patchlist.append((cls, name, old)) + setattr(cls, name, new_meth) + return old + +def _unpatch_meths(patchlist): + for cls, name, old_meth in reversed(patchlist): + setattr(cls, name, old_meth) + +def _patch_for_embedding(patchlist): + if sys.platform == 'win32': + # we must not remove the manifest when building for embedding! + from distutils.msvc9compiler import MSVCCompiler + _patch_meth(patchlist, MSVCCompiler, '_remove_visual_c_ref', + lambda self, manifest_file: manifest_file) + + if sys.platform == 'darwin': + # we must not make a '-bundle', but a '-dynamiclib' instead + from distutils.ccompiler import CCompiler + def my_link_shared_object(self, *args, **kwds): + if '-bundle' in self.linker_so: + self.linker_so = list(self.linker_so) + i = self.linker_so.index('-bundle') + self.linker_so[i] = '-dynamiclib' + return old_link_shared_object(self, *args, **kwds) + old_link_shared_object = _patch_meth(patchlist, CCompiler, + 'link_shared_object', + my_link_shared_object) + +def _patch_for_target(patchlist, target): + from distutils.command.build_ext import build_ext + # if 'target' is different from '*', we need to patch some internal + # method to just return this 'target' value, instead of having it + # built from module_name + if target.endswith('.*'): + target = target[:-2] + if sys.platform == 'win32': + target += '.dll' + elif sys.platform == 'darwin': + target += '.dylib' + else: + target += '.so' + _patch_meth(patchlist, build_ext, 'get_ext_filename', + lambda self, ext_name: target) + + +def recompile(ffi, module_name, preamble, tmpdir='.', call_c_compiler=True, + c_file=None, source_extension='.c', extradir=None, + compiler_verbose=1, target=None, debug=None, **kwds): + if not isinstance(module_name, str): + module_name = module_name.encode('ascii') + if ffi._windows_unicode: + ffi._apply_windows_unicode(kwds) + if preamble is not None: + embedding = (ffi._embedding is not None) + if embedding: + ffi._apply_embedding_fix(kwds) + if c_file is None: + c_file, parts = _modname_to_file(tmpdir, module_name, + source_extension) + if extradir: + parts = [extradir] + parts + ext_c_file = os.path.join(*parts) + else: + ext_c_file = c_file + # + if target is None: + if embedding: + target = '%s.*' % module_name + else: + target = '*' + # + ext = ffiplatform.get_extension(ext_c_file, module_name, **kwds) + updated = make_c_source(ffi, module_name, preamble, c_file, + verbose=compiler_verbose) + if call_c_compiler: + patchlist = [] + cwd = os.getcwd() + try: + if embedding: + _patch_for_embedding(patchlist) + if target != '*': + _patch_for_target(patchlist, target) + if compiler_verbose: + if tmpdir == '.': + msg = 'the current directory is' + else: + msg = 'setting the current directory to' + print('%s %r' % (msg, os.path.abspath(tmpdir))) + os.chdir(tmpdir) + outputfilename = ffiplatform.compile('.', ext, + compiler_verbose, debug) + finally: + os.chdir(cwd) + _unpatch_meths(patchlist) + return outputfilename + else: + return ext, updated + else: + if c_file is None: + c_file, _ = _modname_to_file(tmpdir, module_name, '.py') + updated = make_py_source(ffi, module_name, c_file, + verbose=compiler_verbose) + if call_c_compiler: + return c_file + else: + return None, updated + diff --git a/WebCrawler/venv/Lib/site-packages/cffi/setuptools_ext.py b/WebCrawler/venv/Lib/site-packages/cffi/setuptools_ext.py new file mode 100644 index 0000000..8fe3614 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/cffi/setuptools_ext.py @@ -0,0 +1,219 @@ +import os +import sys + +try: + basestring +except NameError: + # Python 3.x + basestring = str + +def error(msg): + from distutils.errors import DistutilsSetupError + raise DistutilsSetupError(msg) + + +def execfile(filename, glob): + # We use execfile() (here rewritten for Python 3) instead of + # __import__() to load the build script. The problem with + # a normal import is that in some packages, the intermediate + # __init__.py files may already try to import the file that + # we are generating. + with open(filename) as f: + src = f.read() + src += '\n' # Python 2.6 compatibility + code = compile(src, filename, 'exec') + exec(code, glob, glob) + + +def add_cffi_module(dist, mod_spec): + from cffi.api import FFI + + if not isinstance(mod_spec, basestring): + error("argument to 'cffi_modules=...' must be a str or a list of str," + " not %r" % (type(mod_spec).__name__,)) + mod_spec = str(mod_spec) + try: + build_file_name, ffi_var_name = mod_spec.split(':') + except ValueError: + error("%r must be of the form 'path/build.py:ffi_variable'" % + (mod_spec,)) + if not os.path.exists(build_file_name): + ext = '' + rewritten = build_file_name.replace('.', '/') + '.py' + if os.path.exists(rewritten): + ext = ' (rewrite cffi_modules to [%r])' % ( + rewritten + ':' + ffi_var_name,) + error("%r does not name an existing file%s" % (build_file_name, ext)) + + mod_vars = {'__name__': '__cffi__', '__file__': build_file_name} + execfile(build_file_name, mod_vars) + + try: + ffi = mod_vars[ffi_var_name] + except KeyError: + error("%r: object %r not found in module" % (mod_spec, + ffi_var_name)) + if not isinstance(ffi, FFI): + ffi = ffi() # maybe it's a function instead of directly an ffi + if not isinstance(ffi, FFI): + error("%r is not an FFI instance (got %r)" % (mod_spec, + type(ffi).__name__)) + if not hasattr(ffi, '_assigned_source'): + error("%r: the set_source() method was not called" % (mod_spec,)) + module_name, source, source_extension, kwds = ffi._assigned_source + if ffi._windows_unicode: + kwds = kwds.copy() + ffi._apply_windows_unicode(kwds) + + if source is None: + _add_py_module(dist, ffi, module_name) + else: + _add_c_module(dist, ffi, module_name, source, source_extension, kwds) + +def _set_py_limited_api(Extension, kwds): + """ + Add py_limited_api to kwds if setuptools >= 26 is in use. + Do not alter the setting if it already exists. + Setuptools takes care of ignoring the flag on Python 2 and PyPy. + + CPython itself should ignore the flag in a debugging version + (by not listing .abi3.so in the extensions it supports), but + it doesn't so far, creating troubles. That's why we check + for "not hasattr(sys, 'gettotalrefcount')" (the 2.7 compatible equivalent + of 'd' not in sys.abiflags). (http://bugs.python.org/issue28401) + + On Windows, with CPython <= 3.4, it's better not to use py_limited_api + because virtualenv *still* doesn't copy PYTHON3.DLL on these versions. + Recently (2020) we started shipping only >= 3.5 wheels, though. So + we'll give it another try and set py_limited_api on Windows >= 3.5. + """ + from cffi import recompiler + + if ('py_limited_api' not in kwds and not hasattr(sys, 'gettotalrefcount') + and recompiler.USE_LIMITED_API): + import setuptools + try: + setuptools_major_version = int(setuptools.__version__.partition('.')[0]) + if setuptools_major_version >= 26: + kwds['py_limited_api'] = True + except ValueError: # certain development versions of setuptools + # If we don't know the version number of setuptools, we + # try to set 'py_limited_api' anyway. At worst, we get a + # warning. + kwds['py_limited_api'] = True + return kwds + +def _add_c_module(dist, ffi, module_name, source, source_extension, kwds): + from distutils.core import Extension + # We are a setuptools extension. Need this build_ext for py_limited_api. + from setuptools.command.build_ext import build_ext + from distutils.dir_util import mkpath + from distutils import log + from cffi import recompiler + + allsources = ['$PLACEHOLDER'] + allsources.extend(kwds.pop('sources', [])) + kwds = _set_py_limited_api(Extension, kwds) + ext = Extension(name=module_name, sources=allsources, **kwds) + + def make_mod(tmpdir, pre_run=None): + c_file = os.path.join(tmpdir, module_name + source_extension) + log.info("generating cffi module %r" % c_file) + mkpath(tmpdir) + # a setuptools-only, API-only hook: called with the "ext" and "ffi" + # arguments just before we turn the ffi into C code. To use it, + # subclass the 'distutils.command.build_ext.build_ext' class and + # add a method 'def pre_run(self, ext, ffi)'. + if pre_run is not None: + pre_run(ext, ffi) + updated = recompiler.make_c_source(ffi, module_name, source, c_file) + if not updated: + log.info("already up-to-date") + return c_file + + if dist.ext_modules is None: + dist.ext_modules = [] + dist.ext_modules.append(ext) + + base_class = dist.cmdclass.get('build_ext', build_ext) + class build_ext_make_mod(base_class): + def run(self): + if ext.sources[0] == '$PLACEHOLDER': + pre_run = getattr(self, 'pre_run', None) + ext.sources[0] = make_mod(self.build_temp, pre_run) + base_class.run(self) + dist.cmdclass['build_ext'] = build_ext_make_mod + # NB. multiple runs here will create multiple 'build_ext_make_mod' + # classes. Even in this case the 'build_ext' command should be + # run once; but just in case, the logic above does nothing if + # called again. + + +def _add_py_module(dist, ffi, module_name): + from distutils.dir_util import mkpath + from setuptools.command.build_py import build_py + from setuptools.command.build_ext import build_ext + from distutils import log + from cffi import recompiler + + def generate_mod(py_file): + log.info("generating cffi module %r" % py_file) + mkpath(os.path.dirname(py_file)) + updated = recompiler.make_py_source(ffi, module_name, py_file) + if not updated: + log.info("already up-to-date") + + base_class = dist.cmdclass.get('build_py', build_py) + class build_py_make_mod(base_class): + def run(self): + base_class.run(self) + module_path = module_name.split('.') + module_path[-1] += '.py' + generate_mod(os.path.join(self.build_lib, *module_path)) + def get_source_files(self): + # This is called from 'setup.py sdist' only. Exclude + # the generate .py module in this case. + saved_py_modules = self.py_modules + try: + if saved_py_modules: + self.py_modules = [m for m in saved_py_modules + if m != module_name] + return base_class.get_source_files(self) + finally: + self.py_modules = saved_py_modules + dist.cmdclass['build_py'] = build_py_make_mod + + # distutils and setuptools have no notion I could find of a + # generated python module. If we don't add module_name to + # dist.py_modules, then things mostly work but there are some + # combination of options (--root and --record) that will miss + # the module. So we add it here, which gives a few apparently + # harmless warnings about not finding the file outside the + # build directory. + # Then we need to hack more in get_source_files(); see above. + if dist.py_modules is None: + dist.py_modules = [] + dist.py_modules.append(module_name) + + # the following is only for "build_ext -i" + base_class_2 = dist.cmdclass.get('build_ext', build_ext) + class build_ext_make_mod(base_class_2): + def run(self): + base_class_2.run(self) + if self.inplace: + # from get_ext_fullpath() in distutils/command/build_ext.py + module_path = module_name.split('.') + package = '.'.join(module_path[:-1]) + build_py = self.get_finalized_command('build_py') + package_dir = build_py.get_package_dir(package) + file_name = module_path[-1] + '.py' + generate_mod(os.path.join(package_dir, file_name)) + dist.cmdclass['build_ext'] = build_ext_make_mod + +def cffi_modules(dist, attr, value): + assert attr == 'cffi_modules' + if isinstance(value, basestring): + value = [value] + + for cffi_module in value: + add_cffi_module(dist, cffi_module) diff --git a/WebCrawler/venv/Lib/site-packages/cffi/vengine_cpy.py b/WebCrawler/venv/Lib/site-packages/cffi/vengine_cpy.py new file mode 100644 index 0000000..6de0df0 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/cffi/vengine_cpy.py @@ -0,0 +1,1076 @@ +# +# DEPRECATED: implementation for ffi.verify() +# +import sys, imp +from . import model +from .error import VerificationError + + +class VCPythonEngine(object): + _class_key = 'x' + _gen_python_module = True + + def __init__(self, verifier): + self.verifier = verifier + self.ffi = verifier.ffi + self._struct_pending_verification = {} + self._types_of_builtin_functions = {} + + def patch_extension_kwds(self, kwds): + pass + + def find_module(self, module_name, path, so_suffixes): + try: + f, filename, descr = imp.find_module(module_name, path) + except ImportError: + return None + if f is not None: + f.close() + # Note that after a setuptools installation, there are both .py + # and .so files with the same basename. The code here relies on + # imp.find_module() locating the .so in priority. + if descr[0] not in so_suffixes: + return None + return filename + + def collect_types(self): + self._typesdict = {} + self._generate("collecttype") + + def _prnt(self, what=''): + self._f.write(what + '\n') + + def _gettypenum(self, type): + # a KeyError here is a bug. please report it! :-) + return self._typesdict[type] + + def _do_collect_type(self, tp): + if ((not isinstance(tp, model.PrimitiveType) + or tp.name == 'long double') + and tp not in self._typesdict): + num = len(self._typesdict) + self._typesdict[tp] = num + + def write_source_to_f(self): + self.collect_types() + # + # The new module will have a _cffi_setup() function that receives + # objects from the ffi world, and that calls some setup code in + # the module. This setup code is split in several independent + # functions, e.g. one per constant. The functions are "chained" + # by ending in a tail call to each other. + # + # This is further split in two chained lists, depending on if we + # can do it at import-time or if we must wait for _cffi_setup() to + # provide us with the objects. This is needed because we + # need the values of the enum constants in order to build the + # that we may have to pass to _cffi_setup(). + # + # The following two 'chained_list_constants' items contains + # the head of these two chained lists, as a string that gives the + # call to do, if any. + self._chained_list_constants = ['((void)lib,0)', '((void)lib,0)'] + # + prnt = self._prnt + # first paste some standard set of lines that are mostly '#define' + prnt(cffimod_header) + prnt() + # then paste the C source given by the user, verbatim. + prnt(self.verifier.preamble) + prnt() + # + # call generate_cpy_xxx_decl(), for every xxx found from + # ffi._parser._declarations. This generates all the functions. + self._generate("decl") + # + # implement the function _cffi_setup_custom() as calling the + # head of the chained list. + self._generate_setup_custom() + prnt() + # + # produce the method table, including the entries for the + # generated Python->C function wrappers, which are done + # by generate_cpy_function_method(). + prnt('static PyMethodDef _cffi_methods[] = {') + self._generate("method") + prnt(' {"_cffi_setup", _cffi_setup, METH_VARARGS, NULL},') + prnt(' {NULL, NULL, 0, NULL} /* Sentinel */') + prnt('};') + prnt() + # + # standard init. + modname = self.verifier.get_module_name() + constants = self._chained_list_constants[False] + prnt('#if PY_MAJOR_VERSION >= 3') + prnt() + prnt('static struct PyModuleDef _cffi_module_def = {') + prnt(' PyModuleDef_HEAD_INIT,') + prnt(' "%s",' % modname) + prnt(' NULL,') + prnt(' -1,') + prnt(' _cffi_methods,') + prnt(' NULL, NULL, NULL, NULL') + prnt('};') + prnt() + prnt('PyMODINIT_FUNC') + prnt('PyInit_%s(void)' % modname) + prnt('{') + prnt(' PyObject *lib;') + prnt(' lib = PyModule_Create(&_cffi_module_def);') + prnt(' if (lib == NULL)') + prnt(' return NULL;') + prnt(' if (%s < 0 || _cffi_init() < 0) {' % (constants,)) + prnt(' Py_DECREF(lib);') + prnt(' return NULL;') + prnt(' }') + prnt(' return lib;') + prnt('}') + prnt() + prnt('#else') + prnt() + prnt('PyMODINIT_FUNC') + prnt('init%s(void)' % modname) + prnt('{') + prnt(' PyObject *lib;') + prnt(' lib = Py_InitModule("%s", _cffi_methods);' % modname) + prnt(' if (lib == NULL)') + prnt(' return;') + prnt(' if (%s < 0 || _cffi_init() < 0)' % (constants,)) + prnt(' return;') + prnt(' return;') + prnt('}') + prnt() + prnt('#endif') + + def load_library(self, flags=None): + # XXX review all usages of 'self' here! + # import it as a new extension module + imp.acquire_lock() + try: + if hasattr(sys, "getdlopenflags"): + previous_flags = sys.getdlopenflags() + try: + if hasattr(sys, "setdlopenflags") and flags is not None: + sys.setdlopenflags(flags) + module = imp.load_dynamic(self.verifier.get_module_name(), + self.verifier.modulefilename) + except ImportError as e: + error = "importing %r: %s" % (self.verifier.modulefilename, e) + raise VerificationError(error) + finally: + if hasattr(sys, "setdlopenflags"): + sys.setdlopenflags(previous_flags) + finally: + imp.release_lock() + # + # call loading_cpy_struct() to get the struct layout inferred by + # the C compiler + self._load(module, 'loading') + # + # the C code will need the objects. Collect them in + # order in a list. + revmapping = dict([(value, key) + for (key, value) in self._typesdict.items()]) + lst = [revmapping[i] for i in range(len(revmapping))] + lst = list(map(self.ffi._get_cached_btype, lst)) + # + # build the FFILibrary class and instance and call _cffi_setup(). + # this will set up some fields like '_cffi_types', and only then + # it will invoke the chained list of functions that will really + # build (notably) the constant objects, as if they are + # pointers, and store them as attributes on the 'library' object. + class FFILibrary(object): + _cffi_python_module = module + _cffi_ffi = self.ffi + _cffi_dir = [] + def __dir__(self): + return FFILibrary._cffi_dir + list(self.__dict__) + library = FFILibrary() + if module._cffi_setup(lst, VerificationError, library): + import warnings + warnings.warn("reimporting %r might overwrite older definitions" + % (self.verifier.get_module_name())) + # + # finally, call the loaded_cpy_xxx() functions. This will perform + # the final adjustments, like copying the Python->C wrapper + # functions from the module to the 'library' object, and setting + # up the FFILibrary class with properties for the global C variables. + self._load(module, 'loaded', library=library) + module._cffi_original_ffi = self.ffi + module._cffi_types_of_builtin_funcs = self._types_of_builtin_functions + return library + + def _get_declarations(self): + lst = [(key, tp) for (key, (tp, qual)) in + self.ffi._parser._declarations.items()] + lst.sort() + return lst + + def _generate(self, step_name): + for name, tp in self._get_declarations(): + kind, realname = name.split(' ', 1) + try: + method = getattr(self, '_generate_cpy_%s_%s' % (kind, + step_name)) + except AttributeError: + raise VerificationError( + "not implemented in verify(): %r" % name) + try: + method(tp, realname) + except Exception as e: + model.attach_exception_info(e, name) + raise + + def _load(self, module, step_name, **kwds): + for name, tp in self._get_declarations(): + kind, realname = name.split(' ', 1) + method = getattr(self, '_%s_cpy_%s' % (step_name, kind)) + try: + method(tp, realname, module, **kwds) + except Exception as e: + model.attach_exception_info(e, name) + raise + + def _generate_nothing(self, tp, name): + pass + + def _loaded_noop(self, tp, name, module, **kwds): + pass + + # ---------- + + def _convert_funcarg_to_c(self, tp, fromvar, tovar, errcode): + extraarg = '' + if isinstance(tp, model.PrimitiveType): + if tp.is_integer_type() and tp.name != '_Bool': + converter = '_cffi_to_c_int' + extraarg = ', %s' % tp.name + else: + converter = '(%s)_cffi_to_c_%s' % (tp.get_c_name(''), + tp.name.replace(' ', '_')) + errvalue = '-1' + # + elif isinstance(tp, model.PointerType): + self._convert_funcarg_to_c_ptr_or_array(tp, fromvar, + tovar, errcode) + return + # + elif isinstance(tp, (model.StructOrUnion, model.EnumType)): + # a struct (not a struct pointer) as a function argument + self._prnt(' if (_cffi_to_c((char *)&%s, _cffi_type(%d), %s) < 0)' + % (tovar, self._gettypenum(tp), fromvar)) + self._prnt(' %s;' % errcode) + return + # + elif isinstance(tp, model.FunctionPtrType): + converter = '(%s)_cffi_to_c_pointer' % tp.get_c_name('') + extraarg = ', _cffi_type(%d)' % self._gettypenum(tp) + errvalue = 'NULL' + # + else: + raise NotImplementedError(tp) + # + self._prnt(' %s = %s(%s%s);' % (tovar, converter, fromvar, extraarg)) + self._prnt(' if (%s == (%s)%s && PyErr_Occurred())' % ( + tovar, tp.get_c_name(''), errvalue)) + self._prnt(' %s;' % errcode) + + def _extra_local_variables(self, tp, localvars, freelines): + if isinstance(tp, model.PointerType): + localvars.add('Py_ssize_t datasize') + localvars.add('struct _cffi_freeme_s *large_args_free = NULL') + freelines.add('if (large_args_free != NULL)' + ' _cffi_free_array_arguments(large_args_free);') + + def _convert_funcarg_to_c_ptr_or_array(self, tp, fromvar, tovar, errcode): + self._prnt(' datasize = _cffi_prepare_pointer_call_argument(') + self._prnt(' _cffi_type(%d), %s, (char **)&%s);' % ( + self._gettypenum(tp), fromvar, tovar)) + self._prnt(' if (datasize != 0) {') + self._prnt(' %s = ((size_t)datasize) <= 640 ? ' + 'alloca((size_t)datasize) : NULL;' % (tovar,)) + self._prnt(' if (_cffi_convert_array_argument(_cffi_type(%d), %s, ' + '(char **)&%s,' % (self._gettypenum(tp), fromvar, tovar)) + self._prnt(' datasize, &large_args_free) < 0)') + self._prnt(' %s;' % errcode) + self._prnt(' }') + + def _convert_expr_from_c(self, tp, var, context): + if isinstance(tp, model.PrimitiveType): + if tp.is_integer_type() and tp.name != '_Bool': + return '_cffi_from_c_int(%s, %s)' % (var, tp.name) + elif tp.name != 'long double': + return '_cffi_from_c_%s(%s)' % (tp.name.replace(' ', '_'), var) + else: + return '_cffi_from_c_deref((char *)&%s, _cffi_type(%d))' % ( + var, self._gettypenum(tp)) + elif isinstance(tp, (model.PointerType, model.FunctionPtrType)): + return '_cffi_from_c_pointer((char *)%s, _cffi_type(%d))' % ( + var, self._gettypenum(tp)) + elif isinstance(tp, model.ArrayType): + return '_cffi_from_c_pointer((char *)%s, _cffi_type(%d))' % ( + var, self._gettypenum(model.PointerType(tp.item))) + elif isinstance(tp, model.StructOrUnion): + if tp.fldnames is None: + raise TypeError("'%s' is used as %s, but is opaque" % ( + tp._get_c_name(), context)) + return '_cffi_from_c_struct((char *)&%s, _cffi_type(%d))' % ( + var, self._gettypenum(tp)) + elif isinstance(tp, model.EnumType): + return '_cffi_from_c_deref((char *)&%s, _cffi_type(%d))' % ( + var, self._gettypenum(tp)) + else: + raise NotImplementedError(tp) + + # ---------- + # typedefs: generates no code so far + + _generate_cpy_typedef_collecttype = _generate_nothing + _generate_cpy_typedef_decl = _generate_nothing + _generate_cpy_typedef_method = _generate_nothing + _loading_cpy_typedef = _loaded_noop + _loaded_cpy_typedef = _loaded_noop + + # ---------- + # function declarations + + def _generate_cpy_function_collecttype(self, tp, name): + assert isinstance(tp, model.FunctionPtrType) + if tp.ellipsis: + self._do_collect_type(tp) + else: + # don't call _do_collect_type(tp) in this common case, + # otherwise test_autofilled_struct_as_argument fails + for type in tp.args: + self._do_collect_type(type) + self._do_collect_type(tp.result) + + def _generate_cpy_function_decl(self, tp, name): + assert isinstance(tp, model.FunctionPtrType) + if tp.ellipsis: + # cannot support vararg functions better than this: check for its + # exact type (including the fixed arguments), and build it as a + # constant function pointer (no CPython wrapper) + self._generate_cpy_const(False, name, tp) + return + prnt = self._prnt + numargs = len(tp.args) + if numargs == 0: + argname = 'noarg' + elif numargs == 1: + argname = 'arg0' + else: + argname = 'args' + prnt('static PyObject *') + prnt('_cffi_f_%s(PyObject *self, PyObject *%s)' % (name, argname)) + prnt('{') + # + context = 'argument of %s' % name + for i, type in enumerate(tp.args): + prnt(' %s;' % type.get_c_name(' x%d' % i, context)) + # + localvars = set() + freelines = set() + for type in tp.args: + self._extra_local_variables(type, localvars, freelines) + for decl in sorted(localvars): + prnt(' %s;' % (decl,)) + # + if not isinstance(tp.result, model.VoidType): + result_code = 'result = ' + context = 'result of %s' % name + prnt(' %s;' % tp.result.get_c_name(' result', context)) + prnt(' PyObject *pyresult;') + else: + result_code = '' + # + if len(tp.args) > 1: + rng = range(len(tp.args)) + for i in rng: + prnt(' PyObject *arg%d;' % i) + prnt() + prnt(' if (!PyArg_ParseTuple(args, "%s:%s", %s))' % ( + 'O' * numargs, name, ', '.join(['&arg%d' % i for i in rng]))) + prnt(' return NULL;') + prnt() + # + for i, type in enumerate(tp.args): + self._convert_funcarg_to_c(type, 'arg%d' % i, 'x%d' % i, + 'return NULL') + prnt() + # + prnt(' Py_BEGIN_ALLOW_THREADS') + prnt(' _cffi_restore_errno();') + prnt(' { %s%s(%s); }' % ( + result_code, name, + ', '.join(['x%d' % i for i in range(len(tp.args))]))) + prnt(' _cffi_save_errno();') + prnt(' Py_END_ALLOW_THREADS') + prnt() + # + prnt(' (void)self; /* unused */') + if numargs == 0: + prnt(' (void)noarg; /* unused */') + if result_code: + prnt(' pyresult = %s;' % + self._convert_expr_from_c(tp.result, 'result', 'result type')) + for freeline in freelines: + prnt(' ' + freeline) + prnt(' return pyresult;') + else: + for freeline in freelines: + prnt(' ' + freeline) + prnt(' Py_INCREF(Py_None);') + prnt(' return Py_None;') + prnt('}') + prnt() + + def _generate_cpy_function_method(self, tp, name): + if tp.ellipsis: + return + numargs = len(tp.args) + if numargs == 0: + meth = 'METH_NOARGS' + elif numargs == 1: + meth = 'METH_O' + else: + meth = 'METH_VARARGS' + self._prnt(' {"%s", _cffi_f_%s, %s, NULL},' % (name, name, meth)) + + _loading_cpy_function = _loaded_noop + + def _loaded_cpy_function(self, tp, name, module, library): + if tp.ellipsis: + return + func = getattr(module, name) + setattr(library, name, func) + self._types_of_builtin_functions[func] = tp + + # ---------- + # named structs + + _generate_cpy_struct_collecttype = _generate_nothing + def _generate_cpy_struct_decl(self, tp, name): + assert name == tp.name + self._generate_struct_or_union_decl(tp, 'struct', name) + def _generate_cpy_struct_method(self, tp, name): + self._generate_struct_or_union_method(tp, 'struct', name) + def _loading_cpy_struct(self, tp, name, module): + self._loading_struct_or_union(tp, 'struct', name, module) + def _loaded_cpy_struct(self, tp, name, module, **kwds): + self._loaded_struct_or_union(tp) + + _generate_cpy_union_collecttype = _generate_nothing + def _generate_cpy_union_decl(self, tp, name): + assert name == tp.name + self._generate_struct_or_union_decl(tp, 'union', name) + def _generate_cpy_union_method(self, tp, name): + self._generate_struct_or_union_method(tp, 'union', name) + def _loading_cpy_union(self, tp, name, module): + self._loading_struct_or_union(tp, 'union', name, module) + def _loaded_cpy_union(self, tp, name, module, **kwds): + self._loaded_struct_or_union(tp) + + def _generate_struct_or_union_decl(self, tp, prefix, name): + if tp.fldnames is None: + return # nothing to do with opaque structs + checkfuncname = '_cffi_check_%s_%s' % (prefix, name) + layoutfuncname = '_cffi_layout_%s_%s' % (prefix, name) + cname = ('%s %s' % (prefix, name)).strip() + # + prnt = self._prnt + prnt('static void %s(%s *p)' % (checkfuncname, cname)) + prnt('{') + prnt(' /* only to generate compile-time warnings or errors */') + prnt(' (void)p;') + for fname, ftype, fbitsize, fqual in tp.enumfields(): + if (isinstance(ftype, model.PrimitiveType) + and ftype.is_integer_type()) or fbitsize >= 0: + # accept all integers, but complain on float or double + prnt(' (void)((p->%s) << 1);' % fname) + else: + # only accept exactly the type declared. + try: + prnt(' { %s = &p->%s; (void)tmp; }' % ( + ftype.get_c_name('*tmp', 'field %r'%fname, quals=fqual), + fname)) + except VerificationError as e: + prnt(' /* %s */' % str(e)) # cannot verify it, ignore + prnt('}') + prnt('static PyObject *') + prnt('%s(PyObject *self, PyObject *noarg)' % (layoutfuncname,)) + prnt('{') + prnt(' struct _cffi_aligncheck { char x; %s y; };' % cname) + prnt(' static Py_ssize_t nums[] = {') + prnt(' sizeof(%s),' % cname) + prnt(' offsetof(struct _cffi_aligncheck, y),') + for fname, ftype, fbitsize, fqual in tp.enumfields(): + if fbitsize >= 0: + continue # xxx ignore fbitsize for now + prnt(' offsetof(%s, %s),' % (cname, fname)) + if isinstance(ftype, model.ArrayType) and ftype.length is None: + prnt(' 0, /* %s */' % ftype._get_c_name()) + else: + prnt(' sizeof(((%s *)0)->%s),' % (cname, fname)) + prnt(' -1') + prnt(' };') + prnt(' (void)self; /* unused */') + prnt(' (void)noarg; /* unused */') + prnt(' return _cffi_get_struct_layout(nums);') + prnt(' /* the next line is not executed, but compiled */') + prnt(' %s(0);' % (checkfuncname,)) + prnt('}') + prnt() + + def _generate_struct_or_union_method(self, tp, prefix, name): + if tp.fldnames is None: + return # nothing to do with opaque structs + layoutfuncname = '_cffi_layout_%s_%s' % (prefix, name) + self._prnt(' {"%s", %s, METH_NOARGS, NULL},' % (layoutfuncname, + layoutfuncname)) + + def _loading_struct_or_union(self, tp, prefix, name, module): + if tp.fldnames is None: + return # nothing to do with opaque structs + layoutfuncname = '_cffi_layout_%s_%s' % (prefix, name) + # + function = getattr(module, layoutfuncname) + layout = function() + if isinstance(tp, model.StructOrUnion) and tp.partial: + # use the function()'s sizes and offsets to guide the + # layout of the struct + totalsize = layout[0] + totalalignment = layout[1] + fieldofs = layout[2::2] + fieldsize = layout[3::2] + tp.force_flatten() + assert len(fieldofs) == len(fieldsize) == len(tp.fldnames) + tp.fixedlayout = fieldofs, fieldsize, totalsize, totalalignment + else: + cname = ('%s %s' % (prefix, name)).strip() + self._struct_pending_verification[tp] = layout, cname + + def _loaded_struct_or_union(self, tp): + if tp.fldnames is None: + return # nothing to do with opaque structs + self.ffi._get_cached_btype(tp) # force 'fixedlayout' to be considered + + if tp in self._struct_pending_verification: + # check that the layout sizes and offsets match the real ones + def check(realvalue, expectedvalue, msg): + if realvalue != expectedvalue: + raise VerificationError( + "%s (we have %d, but C compiler says %d)" + % (msg, expectedvalue, realvalue)) + ffi = self.ffi + BStruct = ffi._get_cached_btype(tp) + layout, cname = self._struct_pending_verification.pop(tp) + check(layout[0], ffi.sizeof(BStruct), "wrong total size") + check(layout[1], ffi.alignof(BStruct), "wrong total alignment") + i = 2 + for fname, ftype, fbitsize, fqual in tp.enumfields(): + if fbitsize >= 0: + continue # xxx ignore fbitsize for now + check(layout[i], ffi.offsetof(BStruct, fname), + "wrong offset for field %r" % (fname,)) + if layout[i+1] != 0: + BField = ffi._get_cached_btype(ftype) + check(layout[i+1], ffi.sizeof(BField), + "wrong size for field %r" % (fname,)) + i += 2 + assert i == len(layout) + + # ---------- + # 'anonymous' declarations. These are produced for anonymous structs + # or unions; the 'name' is obtained by a typedef. + + _generate_cpy_anonymous_collecttype = _generate_nothing + + def _generate_cpy_anonymous_decl(self, tp, name): + if isinstance(tp, model.EnumType): + self._generate_cpy_enum_decl(tp, name, '') + else: + self._generate_struct_or_union_decl(tp, '', name) + + def _generate_cpy_anonymous_method(self, tp, name): + if not isinstance(tp, model.EnumType): + self._generate_struct_or_union_method(tp, '', name) + + def _loading_cpy_anonymous(self, tp, name, module): + if isinstance(tp, model.EnumType): + self._loading_cpy_enum(tp, name, module) + else: + self._loading_struct_or_union(tp, '', name, module) + + def _loaded_cpy_anonymous(self, tp, name, module, **kwds): + if isinstance(tp, model.EnumType): + self._loaded_cpy_enum(tp, name, module, **kwds) + else: + self._loaded_struct_or_union(tp) + + # ---------- + # constants, likely declared with '#define' + + def _generate_cpy_const(self, is_int, name, tp=None, category='const', + vartp=None, delayed=True, size_too=False, + check_value=None): + prnt = self._prnt + funcname = '_cffi_%s_%s' % (category, name) + prnt('static int %s(PyObject *lib)' % funcname) + prnt('{') + prnt(' PyObject *o;') + prnt(' int res;') + if not is_int: + prnt(' %s;' % (vartp or tp).get_c_name(' i', name)) + else: + assert category == 'const' + # + if check_value is not None: + self._check_int_constant_value(name, check_value) + # + if not is_int: + if category == 'var': + realexpr = '&' + name + else: + realexpr = name + prnt(' i = (%s);' % (realexpr,)) + prnt(' o = %s;' % (self._convert_expr_from_c(tp, 'i', + 'variable type'),)) + assert delayed + else: + prnt(' o = _cffi_from_c_int_const(%s);' % name) + prnt(' if (o == NULL)') + prnt(' return -1;') + if size_too: + prnt(' {') + prnt(' PyObject *o1 = o;') + prnt(' o = Py_BuildValue("On", o1, (Py_ssize_t)sizeof(%s));' + % (name,)) + prnt(' Py_DECREF(o1);') + prnt(' if (o == NULL)') + prnt(' return -1;') + prnt(' }') + prnt(' res = PyObject_SetAttrString(lib, "%s", o);' % name) + prnt(' Py_DECREF(o);') + prnt(' if (res < 0)') + prnt(' return -1;') + prnt(' return %s;' % self._chained_list_constants[delayed]) + self._chained_list_constants[delayed] = funcname + '(lib)' + prnt('}') + prnt() + + def _generate_cpy_constant_collecttype(self, tp, name): + is_int = isinstance(tp, model.PrimitiveType) and tp.is_integer_type() + if not is_int: + self._do_collect_type(tp) + + def _generate_cpy_constant_decl(self, tp, name): + is_int = isinstance(tp, model.PrimitiveType) and tp.is_integer_type() + self._generate_cpy_const(is_int, name, tp) + + _generate_cpy_constant_method = _generate_nothing + _loading_cpy_constant = _loaded_noop + _loaded_cpy_constant = _loaded_noop + + # ---------- + # enums + + def _check_int_constant_value(self, name, value, err_prefix=''): + prnt = self._prnt + if value <= 0: + prnt(' if ((%s) > 0 || (long)(%s) != %dL) {' % ( + name, name, value)) + else: + prnt(' if ((%s) <= 0 || (unsigned long)(%s) != %dUL) {' % ( + name, name, value)) + prnt(' char buf[64];') + prnt(' if ((%s) <= 0)' % name) + prnt(' snprintf(buf, 63, "%%ld", (long)(%s));' % name) + prnt(' else') + prnt(' snprintf(buf, 63, "%%lu", (unsigned long)(%s));' % + name) + prnt(' PyErr_Format(_cffi_VerificationError,') + prnt(' "%s%s has the real value %s, not %s",') + prnt(' "%s", "%s", buf, "%d");' % ( + err_prefix, name, value)) + prnt(' return -1;') + prnt(' }') + + def _enum_funcname(self, prefix, name): + # "$enum_$1" => "___D_enum____D_1" + name = name.replace('$', '___D_') + return '_cffi_e_%s_%s' % (prefix, name) + + def _generate_cpy_enum_decl(self, tp, name, prefix='enum'): + if tp.partial: + for enumerator in tp.enumerators: + self._generate_cpy_const(True, enumerator, delayed=False) + return + # + funcname = self._enum_funcname(prefix, name) + prnt = self._prnt + prnt('static int %s(PyObject *lib)' % funcname) + prnt('{') + for enumerator, enumvalue in zip(tp.enumerators, tp.enumvalues): + self._check_int_constant_value(enumerator, enumvalue, + "enum %s: " % name) + prnt(' return %s;' % self._chained_list_constants[True]) + self._chained_list_constants[True] = funcname + '(lib)' + prnt('}') + prnt() + + _generate_cpy_enum_collecttype = _generate_nothing + _generate_cpy_enum_method = _generate_nothing + + def _loading_cpy_enum(self, tp, name, module): + if tp.partial: + enumvalues = [getattr(module, enumerator) + for enumerator in tp.enumerators] + tp.enumvalues = tuple(enumvalues) + tp.partial_resolved = True + + def _loaded_cpy_enum(self, tp, name, module, library): + for enumerator, enumvalue in zip(tp.enumerators, tp.enumvalues): + setattr(library, enumerator, enumvalue) + + # ---------- + # macros: for now only for integers + + def _generate_cpy_macro_decl(self, tp, name): + if tp == '...': + check_value = None + else: + check_value = tp # an integer + self._generate_cpy_const(True, name, check_value=check_value) + + _generate_cpy_macro_collecttype = _generate_nothing + _generate_cpy_macro_method = _generate_nothing + _loading_cpy_macro = _loaded_noop + _loaded_cpy_macro = _loaded_noop + + # ---------- + # global variables + + def _generate_cpy_variable_collecttype(self, tp, name): + if isinstance(tp, model.ArrayType): + tp_ptr = model.PointerType(tp.item) + else: + tp_ptr = model.PointerType(tp) + self._do_collect_type(tp_ptr) + + def _generate_cpy_variable_decl(self, tp, name): + if isinstance(tp, model.ArrayType): + tp_ptr = model.PointerType(tp.item) + self._generate_cpy_const(False, name, tp, vartp=tp_ptr, + size_too = tp.length_is_unknown()) + else: + tp_ptr = model.PointerType(tp) + self._generate_cpy_const(False, name, tp_ptr, category='var') + + _generate_cpy_variable_method = _generate_nothing + _loading_cpy_variable = _loaded_noop + + def _loaded_cpy_variable(self, tp, name, module, library): + value = getattr(library, name) + if isinstance(tp, model.ArrayType): # int a[5] is "constant" in the + # sense that "a=..." is forbidden + if tp.length_is_unknown(): + assert isinstance(value, tuple) + (value, size) = value + BItemType = self.ffi._get_cached_btype(tp.item) + length, rest = divmod(size, self.ffi.sizeof(BItemType)) + if rest != 0: + raise VerificationError( + "bad size: %r does not seem to be an array of %s" % + (name, tp.item)) + tp = tp.resolve_length(length) + # 'value' is a which we have to replace with + # a if the N is actually known + if tp.length is not None: + BArray = self.ffi._get_cached_btype(tp) + value = self.ffi.cast(BArray, value) + setattr(library, name, value) + return + # remove ptr= from the library instance, and replace + # it by a property on the class, which reads/writes into ptr[0]. + ptr = value + delattr(library, name) + def getter(library): + return ptr[0] + def setter(library, value): + ptr[0] = value + setattr(type(library), name, property(getter, setter)) + type(library)._cffi_dir.append(name) + + # ---------- + + def _generate_setup_custom(self): + prnt = self._prnt + prnt('static int _cffi_setup_custom(PyObject *lib)') + prnt('{') + prnt(' return %s;' % self._chained_list_constants[True]) + prnt('}') + +cffimod_header = r''' +#include +#include + +/* this block of #ifs should be kept exactly identical between + c/_cffi_backend.c, cffi/vengine_cpy.py, cffi/vengine_gen.py + and cffi/_cffi_include.h */ +#if defined(_MSC_VER) +# include /* for alloca() */ +# if _MSC_VER < 1600 /* MSVC < 2010 */ + typedef __int8 int8_t; + typedef __int16 int16_t; + typedef __int32 int32_t; + typedef __int64 int64_t; + typedef unsigned __int8 uint8_t; + typedef unsigned __int16 uint16_t; + typedef unsigned __int32 uint32_t; + typedef unsigned __int64 uint64_t; + typedef __int8 int_least8_t; + typedef __int16 int_least16_t; + typedef __int32 int_least32_t; + typedef __int64 int_least64_t; + typedef unsigned __int8 uint_least8_t; + typedef unsigned __int16 uint_least16_t; + typedef unsigned __int32 uint_least32_t; + typedef unsigned __int64 uint_least64_t; + typedef __int8 int_fast8_t; + typedef __int16 int_fast16_t; + typedef __int32 int_fast32_t; + typedef __int64 int_fast64_t; + typedef unsigned __int8 uint_fast8_t; + typedef unsigned __int16 uint_fast16_t; + typedef unsigned __int32 uint_fast32_t; + typedef unsigned __int64 uint_fast64_t; + typedef __int64 intmax_t; + typedef unsigned __int64 uintmax_t; +# else +# include +# endif +# if _MSC_VER < 1800 /* MSVC < 2013 */ +# ifndef __cplusplus + typedef unsigned char _Bool; +# endif +# endif +#else +# include +# if (defined (__SVR4) && defined (__sun)) || defined(_AIX) || defined(__hpux) +# include +# endif +#endif + +#if PY_MAJOR_VERSION < 3 +# undef PyCapsule_CheckExact +# undef PyCapsule_GetPointer +# define PyCapsule_CheckExact(capsule) (PyCObject_Check(capsule)) +# define PyCapsule_GetPointer(capsule, name) \ + (PyCObject_AsVoidPtr(capsule)) +#endif + +#if PY_MAJOR_VERSION >= 3 +# define PyInt_FromLong PyLong_FromLong +#endif + +#define _cffi_from_c_double PyFloat_FromDouble +#define _cffi_from_c_float PyFloat_FromDouble +#define _cffi_from_c_long PyInt_FromLong +#define _cffi_from_c_ulong PyLong_FromUnsignedLong +#define _cffi_from_c_longlong PyLong_FromLongLong +#define _cffi_from_c_ulonglong PyLong_FromUnsignedLongLong +#define _cffi_from_c__Bool PyBool_FromLong + +#define _cffi_to_c_double PyFloat_AsDouble +#define _cffi_to_c_float PyFloat_AsDouble + +#define _cffi_from_c_int_const(x) \ + (((x) > 0) ? \ + ((unsigned long long)(x) <= (unsigned long long)LONG_MAX) ? \ + PyInt_FromLong((long)(x)) : \ + PyLong_FromUnsignedLongLong((unsigned long long)(x)) : \ + ((long long)(x) >= (long long)LONG_MIN) ? \ + PyInt_FromLong((long)(x)) : \ + PyLong_FromLongLong((long long)(x))) + +#define _cffi_from_c_int(x, type) \ + (((type)-1) > 0 ? /* unsigned */ \ + (sizeof(type) < sizeof(long) ? \ + PyInt_FromLong((long)x) : \ + sizeof(type) == sizeof(long) ? \ + PyLong_FromUnsignedLong((unsigned long)x) : \ + PyLong_FromUnsignedLongLong((unsigned long long)x)) : \ + (sizeof(type) <= sizeof(long) ? \ + PyInt_FromLong((long)x) : \ + PyLong_FromLongLong((long long)x))) + +#define _cffi_to_c_int(o, type) \ + ((type)( \ + sizeof(type) == 1 ? (((type)-1) > 0 ? (type)_cffi_to_c_u8(o) \ + : (type)_cffi_to_c_i8(o)) : \ + sizeof(type) == 2 ? (((type)-1) > 0 ? (type)_cffi_to_c_u16(o) \ + : (type)_cffi_to_c_i16(o)) : \ + sizeof(type) == 4 ? (((type)-1) > 0 ? (type)_cffi_to_c_u32(o) \ + : (type)_cffi_to_c_i32(o)) : \ + sizeof(type) == 8 ? (((type)-1) > 0 ? (type)_cffi_to_c_u64(o) \ + : (type)_cffi_to_c_i64(o)) : \ + (Py_FatalError("unsupported size for type " #type), (type)0))) + +#define _cffi_to_c_i8 \ + ((int(*)(PyObject *))_cffi_exports[1]) +#define _cffi_to_c_u8 \ + ((int(*)(PyObject *))_cffi_exports[2]) +#define _cffi_to_c_i16 \ + ((int(*)(PyObject *))_cffi_exports[3]) +#define _cffi_to_c_u16 \ + ((int(*)(PyObject *))_cffi_exports[4]) +#define _cffi_to_c_i32 \ + ((int(*)(PyObject *))_cffi_exports[5]) +#define _cffi_to_c_u32 \ + ((unsigned int(*)(PyObject *))_cffi_exports[6]) +#define _cffi_to_c_i64 \ + ((long long(*)(PyObject *))_cffi_exports[7]) +#define _cffi_to_c_u64 \ + ((unsigned long long(*)(PyObject *))_cffi_exports[8]) +#define _cffi_to_c_char \ + ((int(*)(PyObject *))_cffi_exports[9]) +#define _cffi_from_c_pointer \ + ((PyObject *(*)(char *, CTypeDescrObject *))_cffi_exports[10]) +#define _cffi_to_c_pointer \ + ((char *(*)(PyObject *, CTypeDescrObject *))_cffi_exports[11]) +#define _cffi_get_struct_layout \ + ((PyObject *(*)(Py_ssize_t[]))_cffi_exports[12]) +#define _cffi_restore_errno \ + ((void(*)(void))_cffi_exports[13]) +#define _cffi_save_errno \ + ((void(*)(void))_cffi_exports[14]) +#define _cffi_from_c_char \ + ((PyObject *(*)(char))_cffi_exports[15]) +#define _cffi_from_c_deref \ + ((PyObject *(*)(char *, CTypeDescrObject *))_cffi_exports[16]) +#define _cffi_to_c \ + ((int(*)(char *, CTypeDescrObject *, PyObject *))_cffi_exports[17]) +#define _cffi_from_c_struct \ + ((PyObject *(*)(char *, CTypeDescrObject *))_cffi_exports[18]) +#define _cffi_to_c_wchar_t \ + ((wchar_t(*)(PyObject *))_cffi_exports[19]) +#define _cffi_from_c_wchar_t \ + ((PyObject *(*)(wchar_t))_cffi_exports[20]) +#define _cffi_to_c_long_double \ + ((long double(*)(PyObject *))_cffi_exports[21]) +#define _cffi_to_c__Bool \ + ((_Bool(*)(PyObject *))_cffi_exports[22]) +#define _cffi_prepare_pointer_call_argument \ + ((Py_ssize_t(*)(CTypeDescrObject *, PyObject *, char **))_cffi_exports[23]) +#define _cffi_convert_array_from_object \ + ((int(*)(char *, CTypeDescrObject *, PyObject *))_cffi_exports[24]) +#define _CFFI_NUM_EXPORTS 25 + +typedef struct _ctypedescr CTypeDescrObject; + +static void *_cffi_exports[_CFFI_NUM_EXPORTS]; +static PyObject *_cffi_types, *_cffi_VerificationError; + +static int _cffi_setup_custom(PyObject *lib); /* forward */ + +static PyObject *_cffi_setup(PyObject *self, PyObject *args) +{ + PyObject *library; + int was_alive = (_cffi_types != NULL); + (void)self; /* unused */ + if (!PyArg_ParseTuple(args, "OOO", &_cffi_types, &_cffi_VerificationError, + &library)) + return NULL; + Py_INCREF(_cffi_types); + Py_INCREF(_cffi_VerificationError); + if (_cffi_setup_custom(library) < 0) + return NULL; + return PyBool_FromLong(was_alive); +} + +union _cffi_union_alignment_u { + unsigned char m_char; + unsigned short m_short; + unsigned int m_int; + unsigned long m_long; + unsigned long long m_longlong; + float m_float; + double m_double; + long double m_longdouble; +}; + +struct _cffi_freeme_s { + struct _cffi_freeme_s *next; + union _cffi_union_alignment_u alignment; +}; + +#ifdef __GNUC__ + __attribute__((unused)) +#endif +static int _cffi_convert_array_argument(CTypeDescrObject *ctptr, PyObject *arg, + char **output_data, Py_ssize_t datasize, + struct _cffi_freeme_s **freeme) +{ + char *p; + if (datasize < 0) + return -1; + + p = *output_data; + if (p == NULL) { + struct _cffi_freeme_s *fp = (struct _cffi_freeme_s *)PyObject_Malloc( + offsetof(struct _cffi_freeme_s, alignment) + (size_t)datasize); + if (fp == NULL) + return -1; + fp->next = *freeme; + *freeme = fp; + p = *output_data = (char *)&fp->alignment; + } + memset((void *)p, 0, (size_t)datasize); + return _cffi_convert_array_from_object(p, ctptr, arg); +} + +#ifdef __GNUC__ + __attribute__((unused)) +#endif +static void _cffi_free_array_arguments(struct _cffi_freeme_s *freeme) +{ + do { + void *p = (void *)freeme; + freeme = freeme->next; + PyObject_Free(p); + } while (freeme != NULL); +} + +static int _cffi_init(void) +{ + PyObject *module, *c_api_object = NULL; + + module = PyImport_ImportModule("_cffi_backend"); + if (module == NULL) + goto failure; + + c_api_object = PyObject_GetAttrString(module, "_C_API"); + if (c_api_object == NULL) + goto failure; + if (!PyCapsule_CheckExact(c_api_object)) { + PyErr_SetNone(PyExc_ImportError); + goto failure; + } + memcpy(_cffi_exports, PyCapsule_GetPointer(c_api_object, "cffi"), + _CFFI_NUM_EXPORTS * sizeof(void *)); + + Py_DECREF(module); + Py_DECREF(c_api_object); + return 0; + + failure: + Py_XDECREF(module); + Py_XDECREF(c_api_object); + return -1; +} + +#define _cffi_type(num) ((CTypeDescrObject *)PyList_GET_ITEM(_cffi_types, num)) + +/**********/ +''' diff --git a/WebCrawler/venv/Lib/site-packages/cffi/vengine_gen.py b/WebCrawler/venv/Lib/site-packages/cffi/vengine_gen.py new file mode 100644 index 0000000..2642152 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/cffi/vengine_gen.py @@ -0,0 +1,675 @@ +# +# DEPRECATED: implementation for ffi.verify() +# +import sys, os +import types + +from . import model +from .error import VerificationError + + +class VGenericEngine(object): + _class_key = 'g' + _gen_python_module = False + + def __init__(self, verifier): + self.verifier = verifier + self.ffi = verifier.ffi + self.export_symbols = [] + self._struct_pending_verification = {} + + def patch_extension_kwds(self, kwds): + # add 'export_symbols' to the dictionary. Note that we add the + # list before filling it. When we fill it, it will thus also show + # up in kwds['export_symbols']. + kwds.setdefault('export_symbols', self.export_symbols) + + def find_module(self, module_name, path, so_suffixes): + for so_suffix in so_suffixes: + basename = module_name + so_suffix + if path is None: + path = sys.path + for dirname in path: + filename = os.path.join(dirname, basename) + if os.path.isfile(filename): + return filename + + def collect_types(self): + pass # not needed in the generic engine + + def _prnt(self, what=''): + self._f.write(what + '\n') + + def write_source_to_f(self): + prnt = self._prnt + # first paste some standard set of lines that are mostly '#include' + prnt(cffimod_header) + # then paste the C source given by the user, verbatim. + prnt(self.verifier.preamble) + # + # call generate_gen_xxx_decl(), for every xxx found from + # ffi._parser._declarations. This generates all the functions. + self._generate('decl') + # + # on Windows, distutils insists on putting init_cffi_xyz in + # 'export_symbols', so instead of fighting it, just give up and + # give it one + if sys.platform == 'win32': + if sys.version_info >= (3,): + prefix = 'PyInit_' + else: + prefix = 'init' + modname = self.verifier.get_module_name() + prnt("void %s%s(void) { }\n" % (prefix, modname)) + + def load_library(self, flags=0): + # import it with the CFFI backend + backend = self.ffi._backend + # needs to make a path that contains '/', on Posix + filename = os.path.join(os.curdir, self.verifier.modulefilename) + module = backend.load_library(filename, flags) + # + # call loading_gen_struct() to get the struct layout inferred by + # the C compiler + self._load(module, 'loading') + + # build the FFILibrary class and instance, this is a module subclass + # because modules are expected to have usually-constant-attributes and + # in PyPy this means the JIT is able to treat attributes as constant, + # which we want. + class FFILibrary(types.ModuleType): + _cffi_generic_module = module + _cffi_ffi = self.ffi + _cffi_dir = [] + def __dir__(self): + return FFILibrary._cffi_dir + library = FFILibrary("") + # + # finally, call the loaded_gen_xxx() functions. This will set + # up the 'library' object. + self._load(module, 'loaded', library=library) + return library + + def _get_declarations(self): + lst = [(key, tp) for (key, (tp, qual)) in + self.ffi._parser._declarations.items()] + lst.sort() + return lst + + def _generate(self, step_name): + for name, tp in self._get_declarations(): + kind, realname = name.split(' ', 1) + try: + method = getattr(self, '_generate_gen_%s_%s' % (kind, + step_name)) + except AttributeError: + raise VerificationError( + "not implemented in verify(): %r" % name) + try: + method(tp, realname) + except Exception as e: + model.attach_exception_info(e, name) + raise + + def _load(self, module, step_name, **kwds): + for name, tp in self._get_declarations(): + kind, realname = name.split(' ', 1) + method = getattr(self, '_%s_gen_%s' % (step_name, kind)) + try: + method(tp, realname, module, **kwds) + except Exception as e: + model.attach_exception_info(e, name) + raise + + def _generate_nothing(self, tp, name): + pass + + def _loaded_noop(self, tp, name, module, **kwds): + pass + + # ---------- + # typedefs: generates no code so far + + _generate_gen_typedef_decl = _generate_nothing + _loading_gen_typedef = _loaded_noop + _loaded_gen_typedef = _loaded_noop + + # ---------- + # function declarations + + def _generate_gen_function_decl(self, tp, name): + assert isinstance(tp, model.FunctionPtrType) + if tp.ellipsis: + # cannot support vararg functions better than this: check for its + # exact type (including the fixed arguments), and build it as a + # constant function pointer (no _cffi_f_%s wrapper) + self._generate_gen_const(False, name, tp) + return + prnt = self._prnt + numargs = len(tp.args) + argnames = [] + for i, type in enumerate(tp.args): + indirection = '' + if isinstance(type, model.StructOrUnion): + indirection = '*' + argnames.append('%sx%d' % (indirection, i)) + context = 'argument of %s' % name + arglist = [type.get_c_name(' %s' % arg, context) + for type, arg in zip(tp.args, argnames)] + tpresult = tp.result + if isinstance(tpresult, model.StructOrUnion): + arglist.insert(0, tpresult.get_c_name(' *r', context)) + tpresult = model.void_type + arglist = ', '.join(arglist) or 'void' + wrappername = '_cffi_f_%s' % name + self.export_symbols.append(wrappername) + if tp.abi: + abi = tp.abi + ' ' + else: + abi = '' + funcdecl = ' %s%s(%s)' % (abi, wrappername, arglist) + context = 'result of %s' % name + prnt(tpresult.get_c_name(funcdecl, context)) + prnt('{') + # + if isinstance(tp.result, model.StructOrUnion): + result_code = '*r = ' + elif not isinstance(tp.result, model.VoidType): + result_code = 'return ' + else: + result_code = '' + prnt(' %s%s(%s);' % (result_code, name, ', '.join(argnames))) + prnt('}') + prnt() + + _loading_gen_function = _loaded_noop + + def _loaded_gen_function(self, tp, name, module, library): + assert isinstance(tp, model.FunctionPtrType) + if tp.ellipsis: + newfunction = self._load_constant(False, tp, name, module) + else: + indirections = [] + base_tp = tp + if (any(isinstance(typ, model.StructOrUnion) for typ in tp.args) + or isinstance(tp.result, model.StructOrUnion)): + indirect_args = [] + for i, typ in enumerate(tp.args): + if isinstance(typ, model.StructOrUnion): + typ = model.PointerType(typ) + indirections.append((i, typ)) + indirect_args.append(typ) + indirect_result = tp.result + if isinstance(indirect_result, model.StructOrUnion): + if indirect_result.fldtypes is None: + raise TypeError("'%s' is used as result type, " + "but is opaque" % ( + indirect_result._get_c_name(),)) + indirect_result = model.PointerType(indirect_result) + indirect_args.insert(0, indirect_result) + indirections.insert(0, ("result", indirect_result)) + indirect_result = model.void_type + tp = model.FunctionPtrType(tuple(indirect_args), + indirect_result, tp.ellipsis) + BFunc = self.ffi._get_cached_btype(tp) + wrappername = '_cffi_f_%s' % name + newfunction = module.load_function(BFunc, wrappername) + for i, typ in indirections: + newfunction = self._make_struct_wrapper(newfunction, i, typ, + base_tp) + setattr(library, name, newfunction) + type(library)._cffi_dir.append(name) + + def _make_struct_wrapper(self, oldfunc, i, tp, base_tp): + backend = self.ffi._backend + BType = self.ffi._get_cached_btype(tp) + if i == "result": + ffi = self.ffi + def newfunc(*args): + res = ffi.new(BType) + oldfunc(res, *args) + return res[0] + else: + def newfunc(*args): + args = args[:i] + (backend.newp(BType, args[i]),) + args[i+1:] + return oldfunc(*args) + newfunc._cffi_base_type = base_tp + return newfunc + + # ---------- + # named structs + + def _generate_gen_struct_decl(self, tp, name): + assert name == tp.name + self._generate_struct_or_union_decl(tp, 'struct', name) + + def _loading_gen_struct(self, tp, name, module): + self._loading_struct_or_union(tp, 'struct', name, module) + + def _loaded_gen_struct(self, tp, name, module, **kwds): + self._loaded_struct_or_union(tp) + + def _generate_gen_union_decl(self, tp, name): + assert name == tp.name + self._generate_struct_or_union_decl(tp, 'union', name) + + def _loading_gen_union(self, tp, name, module): + self._loading_struct_or_union(tp, 'union', name, module) + + def _loaded_gen_union(self, tp, name, module, **kwds): + self._loaded_struct_or_union(tp) + + def _generate_struct_or_union_decl(self, tp, prefix, name): + if tp.fldnames is None: + return # nothing to do with opaque structs + checkfuncname = '_cffi_check_%s_%s' % (prefix, name) + layoutfuncname = '_cffi_layout_%s_%s' % (prefix, name) + cname = ('%s %s' % (prefix, name)).strip() + # + prnt = self._prnt + prnt('static void %s(%s *p)' % (checkfuncname, cname)) + prnt('{') + prnt(' /* only to generate compile-time warnings or errors */') + prnt(' (void)p;') + for fname, ftype, fbitsize, fqual in tp.enumfields(): + if (isinstance(ftype, model.PrimitiveType) + and ftype.is_integer_type()) or fbitsize >= 0: + # accept all integers, but complain on float or double + prnt(' (void)((p->%s) << 1);' % fname) + else: + # only accept exactly the type declared. + try: + prnt(' { %s = &p->%s; (void)tmp; }' % ( + ftype.get_c_name('*tmp', 'field %r'%fname, quals=fqual), + fname)) + except VerificationError as e: + prnt(' /* %s */' % str(e)) # cannot verify it, ignore + prnt('}') + self.export_symbols.append(layoutfuncname) + prnt('intptr_t %s(intptr_t i)' % (layoutfuncname,)) + prnt('{') + prnt(' struct _cffi_aligncheck { char x; %s y; };' % cname) + prnt(' static intptr_t nums[] = {') + prnt(' sizeof(%s),' % cname) + prnt(' offsetof(struct _cffi_aligncheck, y),') + for fname, ftype, fbitsize, fqual in tp.enumfields(): + if fbitsize >= 0: + continue # xxx ignore fbitsize for now + prnt(' offsetof(%s, %s),' % (cname, fname)) + if isinstance(ftype, model.ArrayType) and ftype.length is None: + prnt(' 0, /* %s */' % ftype._get_c_name()) + else: + prnt(' sizeof(((%s *)0)->%s),' % (cname, fname)) + prnt(' -1') + prnt(' };') + prnt(' return nums[i];') + prnt(' /* the next line is not executed, but compiled */') + prnt(' %s(0);' % (checkfuncname,)) + prnt('}') + prnt() + + def _loading_struct_or_union(self, tp, prefix, name, module): + if tp.fldnames is None: + return # nothing to do with opaque structs + layoutfuncname = '_cffi_layout_%s_%s' % (prefix, name) + # + BFunc = self.ffi._typeof_locked("intptr_t(*)(intptr_t)")[0] + function = module.load_function(BFunc, layoutfuncname) + layout = [] + num = 0 + while True: + x = function(num) + if x < 0: break + layout.append(x) + num += 1 + if isinstance(tp, model.StructOrUnion) and tp.partial: + # use the function()'s sizes and offsets to guide the + # layout of the struct + totalsize = layout[0] + totalalignment = layout[1] + fieldofs = layout[2::2] + fieldsize = layout[3::2] + tp.force_flatten() + assert len(fieldofs) == len(fieldsize) == len(tp.fldnames) + tp.fixedlayout = fieldofs, fieldsize, totalsize, totalalignment + else: + cname = ('%s %s' % (prefix, name)).strip() + self._struct_pending_verification[tp] = layout, cname + + def _loaded_struct_or_union(self, tp): + if tp.fldnames is None: + return # nothing to do with opaque structs + self.ffi._get_cached_btype(tp) # force 'fixedlayout' to be considered + + if tp in self._struct_pending_verification: + # check that the layout sizes and offsets match the real ones + def check(realvalue, expectedvalue, msg): + if realvalue != expectedvalue: + raise VerificationError( + "%s (we have %d, but C compiler says %d)" + % (msg, expectedvalue, realvalue)) + ffi = self.ffi + BStruct = ffi._get_cached_btype(tp) + layout, cname = self._struct_pending_verification.pop(tp) + check(layout[0], ffi.sizeof(BStruct), "wrong total size") + check(layout[1], ffi.alignof(BStruct), "wrong total alignment") + i = 2 + for fname, ftype, fbitsize, fqual in tp.enumfields(): + if fbitsize >= 0: + continue # xxx ignore fbitsize for now + check(layout[i], ffi.offsetof(BStruct, fname), + "wrong offset for field %r" % (fname,)) + if layout[i+1] != 0: + BField = ffi._get_cached_btype(ftype) + check(layout[i+1], ffi.sizeof(BField), + "wrong size for field %r" % (fname,)) + i += 2 + assert i == len(layout) + + # ---------- + # 'anonymous' declarations. These are produced for anonymous structs + # or unions; the 'name' is obtained by a typedef. + + def _generate_gen_anonymous_decl(self, tp, name): + if isinstance(tp, model.EnumType): + self._generate_gen_enum_decl(tp, name, '') + else: + self._generate_struct_or_union_decl(tp, '', name) + + def _loading_gen_anonymous(self, tp, name, module): + if isinstance(tp, model.EnumType): + self._loading_gen_enum(tp, name, module, '') + else: + self._loading_struct_or_union(tp, '', name, module) + + def _loaded_gen_anonymous(self, tp, name, module, **kwds): + if isinstance(tp, model.EnumType): + self._loaded_gen_enum(tp, name, module, **kwds) + else: + self._loaded_struct_or_union(tp) + + # ---------- + # constants, likely declared with '#define' + + def _generate_gen_const(self, is_int, name, tp=None, category='const', + check_value=None): + prnt = self._prnt + funcname = '_cffi_%s_%s' % (category, name) + self.export_symbols.append(funcname) + if check_value is not None: + assert is_int + assert category == 'const' + prnt('int %s(char *out_error)' % funcname) + prnt('{') + self._check_int_constant_value(name, check_value) + prnt(' return 0;') + prnt('}') + elif is_int: + assert category == 'const' + prnt('int %s(long long *out_value)' % funcname) + prnt('{') + prnt(' *out_value = (long long)(%s);' % (name,)) + prnt(' return (%s) <= 0;' % (name,)) + prnt('}') + else: + assert tp is not None + assert check_value is None + if category == 'var': + ampersand = '&' + else: + ampersand = '' + extra = '' + if category == 'const' and isinstance(tp, model.StructOrUnion): + extra = 'const *' + ampersand = '&' + prnt(tp.get_c_name(' %s%s(void)' % (extra, funcname), name)) + prnt('{') + prnt(' return (%s%s);' % (ampersand, name)) + prnt('}') + prnt() + + def _generate_gen_constant_decl(self, tp, name): + is_int = isinstance(tp, model.PrimitiveType) and tp.is_integer_type() + self._generate_gen_const(is_int, name, tp) + + _loading_gen_constant = _loaded_noop + + def _load_constant(self, is_int, tp, name, module, check_value=None): + funcname = '_cffi_const_%s' % name + if check_value is not None: + assert is_int + self._load_known_int_constant(module, funcname) + value = check_value + elif is_int: + BType = self.ffi._typeof_locked("long long*")[0] + BFunc = self.ffi._typeof_locked("int(*)(long long*)")[0] + function = module.load_function(BFunc, funcname) + p = self.ffi.new(BType) + negative = function(p) + value = int(p[0]) + if value < 0 and not negative: + BLongLong = self.ffi._typeof_locked("long long")[0] + value += (1 << (8*self.ffi.sizeof(BLongLong))) + else: + assert check_value is None + fntypeextra = '(*)(void)' + if isinstance(tp, model.StructOrUnion): + fntypeextra = '*' + fntypeextra + BFunc = self.ffi._typeof_locked(tp.get_c_name(fntypeextra, name))[0] + function = module.load_function(BFunc, funcname) + value = function() + if isinstance(tp, model.StructOrUnion): + value = value[0] + return value + + def _loaded_gen_constant(self, tp, name, module, library): + is_int = isinstance(tp, model.PrimitiveType) and tp.is_integer_type() + value = self._load_constant(is_int, tp, name, module) + setattr(library, name, value) + type(library)._cffi_dir.append(name) + + # ---------- + # enums + + def _check_int_constant_value(self, name, value): + prnt = self._prnt + if value <= 0: + prnt(' if ((%s) > 0 || (long)(%s) != %dL) {' % ( + name, name, value)) + else: + prnt(' if ((%s) <= 0 || (unsigned long)(%s) != %dUL) {' % ( + name, name, value)) + prnt(' char buf[64];') + prnt(' if ((%s) <= 0)' % name) + prnt(' sprintf(buf, "%%ld", (long)(%s));' % name) + prnt(' else') + prnt(' sprintf(buf, "%%lu", (unsigned long)(%s));' % + name) + prnt(' sprintf(out_error, "%s has the real value %s, not %s",') + prnt(' "%s", buf, "%d");' % (name[:100], value)) + prnt(' return -1;') + prnt(' }') + + def _load_known_int_constant(self, module, funcname): + BType = self.ffi._typeof_locked("char[]")[0] + BFunc = self.ffi._typeof_locked("int(*)(char*)")[0] + function = module.load_function(BFunc, funcname) + p = self.ffi.new(BType, 256) + if function(p) < 0: + error = self.ffi.string(p) + if sys.version_info >= (3,): + error = str(error, 'utf-8') + raise VerificationError(error) + + def _enum_funcname(self, prefix, name): + # "$enum_$1" => "___D_enum____D_1" + name = name.replace('$', '___D_') + return '_cffi_e_%s_%s' % (prefix, name) + + def _generate_gen_enum_decl(self, tp, name, prefix='enum'): + if tp.partial: + for enumerator in tp.enumerators: + self._generate_gen_const(True, enumerator) + return + # + funcname = self._enum_funcname(prefix, name) + self.export_symbols.append(funcname) + prnt = self._prnt + prnt('int %s(char *out_error)' % funcname) + prnt('{') + for enumerator, enumvalue in zip(tp.enumerators, tp.enumvalues): + self._check_int_constant_value(enumerator, enumvalue) + prnt(' return 0;') + prnt('}') + prnt() + + def _loading_gen_enum(self, tp, name, module, prefix='enum'): + if tp.partial: + enumvalues = [self._load_constant(True, tp, enumerator, module) + for enumerator in tp.enumerators] + tp.enumvalues = tuple(enumvalues) + tp.partial_resolved = True + else: + funcname = self._enum_funcname(prefix, name) + self._load_known_int_constant(module, funcname) + + def _loaded_gen_enum(self, tp, name, module, library): + for enumerator, enumvalue in zip(tp.enumerators, tp.enumvalues): + setattr(library, enumerator, enumvalue) + type(library)._cffi_dir.append(enumerator) + + # ---------- + # macros: for now only for integers + + def _generate_gen_macro_decl(self, tp, name): + if tp == '...': + check_value = None + else: + check_value = tp # an integer + self._generate_gen_const(True, name, check_value=check_value) + + _loading_gen_macro = _loaded_noop + + def _loaded_gen_macro(self, tp, name, module, library): + if tp == '...': + check_value = None + else: + check_value = tp # an integer + value = self._load_constant(True, tp, name, module, + check_value=check_value) + setattr(library, name, value) + type(library)._cffi_dir.append(name) + + # ---------- + # global variables + + def _generate_gen_variable_decl(self, tp, name): + if isinstance(tp, model.ArrayType): + if tp.length_is_unknown(): + prnt = self._prnt + funcname = '_cffi_sizeof_%s' % (name,) + self.export_symbols.append(funcname) + prnt("size_t %s(void)" % funcname) + prnt("{") + prnt(" return sizeof(%s);" % (name,)) + prnt("}") + tp_ptr = model.PointerType(tp.item) + self._generate_gen_const(False, name, tp_ptr) + else: + tp_ptr = model.PointerType(tp) + self._generate_gen_const(False, name, tp_ptr, category='var') + + _loading_gen_variable = _loaded_noop + + def _loaded_gen_variable(self, tp, name, module, library): + if isinstance(tp, model.ArrayType): # int a[5] is "constant" in the + # sense that "a=..." is forbidden + if tp.length_is_unknown(): + funcname = '_cffi_sizeof_%s' % (name,) + BFunc = self.ffi._typeof_locked('size_t(*)(void)')[0] + function = module.load_function(BFunc, funcname) + size = function() + BItemType = self.ffi._get_cached_btype(tp.item) + length, rest = divmod(size, self.ffi.sizeof(BItemType)) + if rest != 0: + raise VerificationError( + "bad size: %r does not seem to be an array of %s" % + (name, tp.item)) + tp = tp.resolve_length(length) + tp_ptr = model.PointerType(tp.item) + value = self._load_constant(False, tp_ptr, name, module) + # 'value' is a which we have to replace with + # a if the N is actually known + if tp.length is not None: + BArray = self.ffi._get_cached_btype(tp) + value = self.ffi.cast(BArray, value) + setattr(library, name, value) + type(library)._cffi_dir.append(name) + return + # remove ptr= from the library instance, and replace + # it by a property on the class, which reads/writes into ptr[0]. + funcname = '_cffi_var_%s' % name + BFunc = self.ffi._typeof_locked(tp.get_c_name('*(*)(void)', name))[0] + function = module.load_function(BFunc, funcname) + ptr = function() + def getter(library): + return ptr[0] + def setter(library, value): + ptr[0] = value + setattr(type(library), name, property(getter, setter)) + type(library)._cffi_dir.append(name) + +cffimod_header = r''' +#include +#include +#include +#include +#include /* XXX for ssize_t on some platforms */ + +/* this block of #ifs should be kept exactly identical between + c/_cffi_backend.c, cffi/vengine_cpy.py, cffi/vengine_gen.py + and cffi/_cffi_include.h */ +#if defined(_MSC_VER) +# include /* for alloca() */ +# if _MSC_VER < 1600 /* MSVC < 2010 */ + typedef __int8 int8_t; + typedef __int16 int16_t; + typedef __int32 int32_t; + typedef __int64 int64_t; + typedef unsigned __int8 uint8_t; + typedef unsigned __int16 uint16_t; + typedef unsigned __int32 uint32_t; + typedef unsigned __int64 uint64_t; + typedef __int8 int_least8_t; + typedef __int16 int_least16_t; + typedef __int32 int_least32_t; + typedef __int64 int_least64_t; + typedef unsigned __int8 uint_least8_t; + typedef unsigned __int16 uint_least16_t; + typedef unsigned __int32 uint_least32_t; + typedef unsigned __int64 uint_least64_t; + typedef __int8 int_fast8_t; + typedef __int16 int_fast16_t; + typedef __int32 int_fast32_t; + typedef __int64 int_fast64_t; + typedef unsigned __int8 uint_fast8_t; + typedef unsigned __int16 uint_fast16_t; + typedef unsigned __int32 uint_fast32_t; + typedef unsigned __int64 uint_fast64_t; + typedef __int64 intmax_t; + typedef unsigned __int64 uintmax_t; +# else +# include +# endif +# if _MSC_VER < 1800 /* MSVC < 2013 */ +# ifndef __cplusplus + typedef unsigned char _Bool; +# endif +# endif +#else +# include +# if (defined (__SVR4) && defined (__sun)) || defined(_AIX) || defined(__hpux) +# include +# endif +#endif +''' diff --git a/WebCrawler/venv/Lib/site-packages/cffi/verifier.py b/WebCrawler/venv/Lib/site-packages/cffi/verifier.py new file mode 100644 index 0000000..59b78c2 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/cffi/verifier.py @@ -0,0 +1,306 @@ +# +# DEPRECATED: implementation for ffi.verify() +# +import sys, os, binascii, shutil, io +from . import __version_verifier_modules__ +from . import ffiplatform +from .error import VerificationError + +if sys.version_info >= (3, 3): + import importlib.machinery + def _extension_suffixes(): + return importlib.machinery.EXTENSION_SUFFIXES[:] +else: + import imp + def _extension_suffixes(): + return [suffix for suffix, _, type in imp.get_suffixes() + if type == imp.C_EXTENSION] + + +if sys.version_info >= (3,): + NativeIO = io.StringIO +else: + class NativeIO(io.BytesIO): + def write(self, s): + if isinstance(s, unicode): + s = s.encode('ascii') + super(NativeIO, self).write(s) + + +class Verifier(object): + + def __init__(self, ffi, preamble, tmpdir=None, modulename=None, + ext_package=None, tag='', force_generic_engine=False, + source_extension='.c', flags=None, relative_to=None, **kwds): + if ffi._parser._uses_new_feature: + raise VerificationError( + "feature not supported with ffi.verify(), but only " + "with ffi.set_source(): %s" % (ffi._parser._uses_new_feature,)) + self.ffi = ffi + self.preamble = preamble + if not modulename: + flattened_kwds = ffiplatform.flatten(kwds) + vengine_class = _locate_engine_class(ffi, force_generic_engine) + self._vengine = vengine_class(self) + self._vengine.patch_extension_kwds(kwds) + self.flags = flags + self.kwds = self.make_relative_to(kwds, relative_to) + # + if modulename: + if tag: + raise TypeError("can't specify both 'modulename' and 'tag'") + else: + key = '\x00'.join([sys.version[:3], __version_verifier_modules__, + preamble, flattened_kwds] + + ffi._cdefsources) + if sys.version_info >= (3,): + key = key.encode('utf-8') + k1 = hex(binascii.crc32(key[0::2]) & 0xffffffff) + k1 = k1.lstrip('0x').rstrip('L') + k2 = hex(binascii.crc32(key[1::2]) & 0xffffffff) + k2 = k2.lstrip('0').rstrip('L') + modulename = '_cffi_%s_%s%s%s' % (tag, self._vengine._class_key, + k1, k2) + suffix = _get_so_suffixes()[0] + self.tmpdir = tmpdir or _caller_dir_pycache() + self.sourcefilename = os.path.join(self.tmpdir, modulename + source_extension) + self.modulefilename = os.path.join(self.tmpdir, modulename + suffix) + self.ext_package = ext_package + self._has_source = False + self._has_module = False + + def write_source(self, file=None): + """Write the C source code. It is produced in 'self.sourcefilename', + which can be tweaked beforehand.""" + with self.ffi._lock: + if self._has_source and file is None: + raise VerificationError( + "source code already written") + self._write_source(file) + + def compile_module(self): + """Write the C source code (if not done already) and compile it. + This produces a dynamic link library in 'self.modulefilename'.""" + with self.ffi._lock: + if self._has_module: + raise VerificationError("module already compiled") + if not self._has_source: + self._write_source() + self._compile_module() + + def load_library(self): + """Get a C module from this Verifier instance. + Returns an instance of a FFILibrary class that behaves like the + objects returned by ffi.dlopen(), but that delegates all + operations to the C module. If necessary, the C code is written + and compiled first. + """ + with self.ffi._lock: + if not self._has_module: + self._locate_module() + if not self._has_module: + if not self._has_source: + self._write_source() + self._compile_module() + return self._load_library() + + def get_module_name(self): + basename = os.path.basename(self.modulefilename) + # kill both the .so extension and the other .'s, as introduced + # by Python 3: 'basename.cpython-33m.so' + basename = basename.split('.', 1)[0] + # and the _d added in Python 2 debug builds --- but try to be + # conservative and not kill a legitimate _d + if basename.endswith('_d') and hasattr(sys, 'gettotalrefcount'): + basename = basename[:-2] + return basename + + def get_extension(self): + ffiplatform._hack_at_distutils() # backward compatibility hack + if not self._has_source: + with self.ffi._lock: + if not self._has_source: + self._write_source() + sourcename = ffiplatform.maybe_relative_path(self.sourcefilename) + modname = self.get_module_name() + return ffiplatform.get_extension(sourcename, modname, **self.kwds) + + def generates_python_module(self): + return self._vengine._gen_python_module + + def make_relative_to(self, kwds, relative_to): + if relative_to and os.path.dirname(relative_to): + dirname = os.path.dirname(relative_to) + kwds = kwds.copy() + for key in ffiplatform.LIST_OF_FILE_NAMES: + if key in kwds: + lst = kwds[key] + if not isinstance(lst, (list, tuple)): + raise TypeError("keyword '%s' should be a list or tuple" + % (key,)) + lst = [os.path.join(dirname, fn) for fn in lst] + kwds[key] = lst + return kwds + + # ---------- + + def _locate_module(self): + if not os.path.isfile(self.modulefilename): + if self.ext_package: + try: + pkg = __import__(self.ext_package, None, None, ['__doc__']) + except ImportError: + return # cannot import the package itself, give up + # (e.g. it might be called differently before installation) + path = pkg.__path__ + else: + path = None + filename = self._vengine.find_module(self.get_module_name(), path, + _get_so_suffixes()) + if filename is None: + return + self.modulefilename = filename + self._vengine.collect_types() + self._has_module = True + + def _write_source_to(self, file): + self._vengine._f = file + try: + self._vengine.write_source_to_f() + finally: + del self._vengine._f + + def _write_source(self, file=None): + if file is not None: + self._write_source_to(file) + else: + # Write our source file to an in memory file. + f = NativeIO() + self._write_source_to(f) + source_data = f.getvalue() + + # Determine if this matches the current file + if os.path.exists(self.sourcefilename): + with open(self.sourcefilename, "r") as fp: + needs_written = not (fp.read() == source_data) + else: + needs_written = True + + # Actually write the file out if it doesn't match + if needs_written: + _ensure_dir(self.sourcefilename) + with open(self.sourcefilename, "w") as fp: + fp.write(source_data) + + # Set this flag + self._has_source = True + + def _compile_module(self): + # compile this C source + tmpdir = os.path.dirname(self.sourcefilename) + outputfilename = ffiplatform.compile(tmpdir, self.get_extension()) + try: + same = ffiplatform.samefile(outputfilename, self.modulefilename) + except OSError: + same = False + if not same: + _ensure_dir(self.modulefilename) + shutil.move(outputfilename, self.modulefilename) + self._has_module = True + + def _load_library(self): + assert self._has_module + if self.flags is not None: + return self._vengine.load_library(self.flags) + else: + return self._vengine.load_library() + +# ____________________________________________________________ + +_FORCE_GENERIC_ENGINE = False # for tests + +def _locate_engine_class(ffi, force_generic_engine): + if _FORCE_GENERIC_ENGINE: + force_generic_engine = True + if not force_generic_engine: + if '__pypy__' in sys.builtin_module_names: + force_generic_engine = True + else: + try: + import _cffi_backend + except ImportError: + _cffi_backend = '?' + if ffi._backend is not _cffi_backend: + force_generic_engine = True + if force_generic_engine: + from . import vengine_gen + return vengine_gen.VGenericEngine + else: + from . import vengine_cpy + return vengine_cpy.VCPythonEngine + +# ____________________________________________________________ + +_TMPDIR = None + +def _caller_dir_pycache(): + if _TMPDIR: + return _TMPDIR + result = os.environ.get('CFFI_TMPDIR') + if result: + return result + filename = sys._getframe(2).f_code.co_filename + return os.path.abspath(os.path.join(os.path.dirname(filename), + '__pycache__')) + +def set_tmpdir(dirname): + """Set the temporary directory to use instead of __pycache__.""" + global _TMPDIR + _TMPDIR = dirname + +def cleanup_tmpdir(tmpdir=None, keep_so=False): + """Clean up the temporary directory by removing all files in it + called `_cffi_*.{c,so}` as well as the `build` subdirectory.""" + tmpdir = tmpdir or _caller_dir_pycache() + try: + filelist = os.listdir(tmpdir) + except OSError: + return + if keep_so: + suffix = '.c' # only remove .c files + else: + suffix = _get_so_suffixes()[0].lower() + for fn in filelist: + if fn.lower().startswith('_cffi_') and ( + fn.lower().endswith(suffix) or fn.lower().endswith('.c')): + try: + os.unlink(os.path.join(tmpdir, fn)) + except OSError: + pass + clean_dir = [os.path.join(tmpdir, 'build')] + for dir in clean_dir: + try: + for fn in os.listdir(dir): + fn = os.path.join(dir, fn) + if os.path.isdir(fn): + clean_dir.append(fn) + else: + os.unlink(fn) + except OSError: + pass + +def _get_so_suffixes(): + suffixes = _extension_suffixes() + if not suffixes: + # bah, no C_EXTENSION available. Occurs on pypy without cpyext + if sys.platform == 'win32': + suffixes = [".pyd"] + else: + suffixes = [".so"] + + return suffixes + +def _ensure_dir(filename): + dirname = os.path.dirname(filename) + if dirname and not os.path.isdir(dirname): + os.makedirs(dirname) diff --git a/WebCrawler/venv/Lib/site-packages/colorama-0.4.4.dist-info/INSTALLER b/WebCrawler/venv/Lib/site-packages/colorama-0.4.4.dist-info/INSTALLER new file mode 100644 index 0000000..a1b589e --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/colorama-0.4.4.dist-info/INSTALLER @@ -0,0 +1 @@ +pip diff --git a/WebCrawler/venv/Lib/site-packages/colorama-0.4.4.dist-info/LICENSE.txt b/WebCrawler/venv/Lib/site-packages/colorama-0.4.4.dist-info/LICENSE.txt new file mode 100644 index 0000000..3105888 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/colorama-0.4.4.dist-info/LICENSE.txt @@ -0,0 +1,27 @@ +Copyright (c) 2010 Jonathan Hartley +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +* Neither the name of the copyright holders, nor those of its contributors + may be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/WebCrawler/venv/Lib/site-packages/colorama-0.4.4.dist-info/METADATA b/WebCrawler/venv/Lib/site-packages/colorama-0.4.4.dist-info/METADATA new file mode 100644 index 0000000..2a175c2 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/colorama-0.4.4.dist-info/METADATA @@ -0,0 +1,415 @@ +Metadata-Version: 2.1 +Name: colorama +Version: 0.4.4 +Summary: Cross-platform colored terminal text. +Home-page: https://github.com/tartley/colorama +Author: Jonathan Hartley +Author-email: tartley@tartley.com +Maintainer: Arnon Yaari +License: BSD +Keywords: color colour terminal text ansi windows crossplatform xplatform +Platform: UNKNOWN +Classifier: Development Status :: 5 - Production/Stable +Classifier: Environment :: Console +Classifier: Intended Audience :: Developers +Classifier: License :: OSI Approved :: BSD License +Classifier: Operating System :: OS Independent +Classifier: Programming Language :: Python +Classifier: Programming Language :: Python :: 2 +Classifier: Programming Language :: Python :: 2.7 +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3.5 +Classifier: Programming Language :: Python :: 3.6 +Classifier: Programming Language :: Python :: 3.7 +Classifier: Programming Language :: Python :: 3.8 +Classifier: Programming Language :: Python :: Implementation :: CPython +Classifier: Programming Language :: Python :: Implementation :: PyPy +Classifier: Topic :: Terminals +Requires-Python: >=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.* + +.. image:: https://img.shields.io/pypi/v/colorama.svg + :target: https://pypi.org/project/colorama/ + :alt: Latest Version + +.. image:: https://img.shields.io/pypi/pyversions/colorama.svg + :target: https://pypi.org/project/colorama/ + :alt: Supported Python versions + +.. image:: https://travis-ci.org/tartley/colorama.svg?branch=master + :target: https://travis-ci.org/tartley/colorama + :alt: Build Status + +Colorama +======== + +Makes ANSI escape character sequences (for producing colored terminal text and +cursor positioning) work under MS Windows. + +.. |donate| image:: https://www.paypalobjects.com/en_US/i/btn/btn_donate_SM.gif + :target: https://www.paypal.com/cgi-bin/webscr?cmd=_donations&business=2MZ9D2GMLYCUJ&item_name=Colorama¤cy_code=USD + :alt: Donate with Paypal + +`PyPI for releases `_ · +`Github for source `_ · +`Colorama for enterprise on Tidelift `_ + +If you find Colorama useful, please |donate| to the authors. Thank you! + + +Installation +------------ + +.. code-block:: bash + + pip install colorama + # or + conda install -c anaconda colorama + + +Description +----------- + +ANSI escape character sequences have long been used to produce colored terminal +text and cursor positioning on Unix and Macs. Colorama makes this work on +Windows, too, by wrapping ``stdout``, stripping ANSI sequences it finds (which +would appear as gobbledygook in the output), and converting them into the +appropriate win32 calls to modify the state of the terminal. On other platforms, +Colorama does nothing. + +This has the upshot of providing a simple cross-platform API for printing +colored terminal text from Python, and has the happy side-effect that existing +applications or libraries which use ANSI sequences to produce colored output on +Linux or Macs can now also work on Windows, simply by calling +``colorama.init()``. + +An alternative approach is to install ``ansi.sys`` on Windows machines, which +provides the same behaviour for all applications running in terminals. Colorama +is intended for situations where that isn't easy (e.g., maybe your app doesn't +have an installer.) + +Demo scripts in the source code repository print some colored text using +ANSI sequences. Compare their output under Gnome-terminal's built in ANSI +handling, versus on Windows Command-Prompt using Colorama: + +.. image:: https://github.com/tartley/colorama/raw/master/screenshots/ubuntu-demo.png + :width: 661 + :height: 357 + :alt: ANSI sequences on Ubuntu under gnome-terminal. + +.. image:: https://github.com/tartley/colorama/raw/master/screenshots/windows-demo.png + :width: 668 + :height: 325 + :alt: Same ANSI sequences on Windows, using Colorama. + +These screenshots show that, on Windows, Colorama does not support ANSI 'dim +text'; it looks the same as 'normal text'. + +Usage +----- + +Initialisation +.............. + +Applications should initialise Colorama using: + +.. code-block:: python + + from colorama import init + init() + +On Windows, calling ``init()`` will filter ANSI escape sequences out of any +text sent to ``stdout`` or ``stderr``, and replace them with equivalent Win32 +calls. + +On other platforms, calling ``init()`` has no effect (unless you request other +optional functionality; see "Init Keyword Args", below). By design, this permits +applications to call ``init()`` unconditionally on all platforms, after which +ANSI output should just work. + +To stop using Colorama before your program exits, simply call ``deinit()``. +This will restore ``stdout`` and ``stderr`` to their original values, so that +Colorama is disabled. To resume using Colorama again, call ``reinit()``; it is +cheaper than calling ``init()`` again (but does the same thing). + + +Colored Output +.............. + +Cross-platform printing of colored text can then be done using Colorama's +constant shorthand for ANSI escape sequences: + +.. code-block:: python + + from colorama import Fore, Back, Style + print(Fore.RED + 'some red text') + print(Back.GREEN + 'and with a green background') + print(Style.DIM + 'and in dim text') + print(Style.RESET_ALL) + print('back to normal now') + +...or simply by manually printing ANSI sequences from your own code: + +.. code-block:: python + + print('\033[31m' + 'some red text') + print('\033[39m') # and reset to default color + +...or, Colorama can be used in conjunction with existing ANSI libraries +such as the venerable `Termcolor `_ +or the fabulous `Blessings `_. +This is highly recommended for anything more than trivial coloring: + +.. code-block:: python + + from colorama import init + from termcolor import colored + + # use Colorama to make Termcolor work on Windows too + init() + + # then use Termcolor for all colored text output + print(colored('Hello, World!', 'green', 'on_red')) + +Available formatting constants are:: + + Fore: BLACK, RED, GREEN, YELLOW, BLUE, MAGENTA, CYAN, WHITE, RESET. + Back: BLACK, RED, GREEN, YELLOW, BLUE, MAGENTA, CYAN, WHITE, RESET. + Style: DIM, NORMAL, BRIGHT, RESET_ALL + +``Style.RESET_ALL`` resets foreground, background, and brightness. Colorama will +perform this reset automatically on program exit. + + +Cursor Positioning +.................. + +ANSI codes to reposition the cursor are supported. See ``demos/demo06.py`` for +an example of how to generate them. + + +Init Keyword Args +................. + +``init()`` accepts some ``**kwargs`` to override default behaviour. + +init(autoreset=False): + If you find yourself repeatedly sending reset sequences to turn off color + changes at the end of every print, then ``init(autoreset=True)`` will + automate that: + + .. code-block:: python + + from colorama import init + init(autoreset=True) + print(Fore.RED + 'some red text') + print('automatically back to default color again') + +init(strip=None): + Pass ``True`` or ``False`` to override whether ANSI codes should be + stripped from the output. The default behaviour is to strip if on Windows + or if output is redirected (not a tty). + +init(convert=None): + Pass ``True`` or ``False`` to override whether to convert ANSI codes in the + output into win32 calls. The default behaviour is to convert if on Windows + and output is to a tty (terminal). + +init(wrap=True): + On Windows, Colorama works by replacing ``sys.stdout`` and ``sys.stderr`` + with proxy objects, which override the ``.write()`` method to do their work. + If this wrapping causes you problems, then this can be disabled by passing + ``init(wrap=False)``. The default behaviour is to wrap if ``autoreset`` or + ``strip`` or ``convert`` are True. + + When wrapping is disabled, colored printing on non-Windows platforms will + continue to work as normal. To do cross-platform colored output, you can + use Colorama's ``AnsiToWin32`` proxy directly: + + .. code-block:: python + + import sys + from colorama import init, AnsiToWin32 + init(wrap=False) + stream = AnsiToWin32(sys.stderr).stream + + # Python 2 + print >>stream, Fore.BLUE + 'blue text on stderr' + + # Python 3 + print(Fore.BLUE + 'blue text on stderr', file=stream) + + +Recognised ANSI Sequences +......................... + +ANSI sequences generally take the form:: + + ESC [ ; ... + +Where ```` is an integer, and ```` is a single letter. Zero or +more params are passed to a ````. If no params are passed, it is +generally synonymous with passing a single zero. No spaces exist in the +sequence; they have been inserted here simply to read more easily. + +The only ANSI sequences that Colorama converts into win32 calls are:: + + ESC [ 0 m # reset all (colors and brightness) + ESC [ 1 m # bright + ESC [ 2 m # dim (looks same as normal brightness) + ESC [ 22 m # normal brightness + + # FOREGROUND: + ESC [ 30 m # black + ESC [ 31 m # red + ESC [ 32 m # green + ESC [ 33 m # yellow + ESC [ 34 m # blue + ESC [ 35 m # magenta + ESC [ 36 m # cyan + ESC [ 37 m # white + ESC [ 39 m # reset + + # BACKGROUND + ESC [ 40 m # black + ESC [ 41 m # red + ESC [ 42 m # green + ESC [ 43 m # yellow + ESC [ 44 m # blue + ESC [ 45 m # magenta + ESC [ 46 m # cyan + ESC [ 47 m # white + ESC [ 49 m # reset + + # cursor positioning + ESC [ y;x H # position cursor at x across, y down + ESC [ y;x f # position cursor at x across, y down + ESC [ n A # move cursor n lines up + ESC [ n B # move cursor n lines down + ESC [ n C # move cursor n characters forward + ESC [ n D # move cursor n characters backward + + # clear the screen + ESC [ mode J # clear the screen + + # clear the line + ESC [ mode K # clear the line + +Multiple numeric params to the ``'m'`` command can be combined into a single +sequence:: + + ESC [ 36 ; 45 ; 1 m # bright cyan text on magenta background + +All other ANSI sequences of the form ``ESC [ ; ... `` +are silently stripped from the output on Windows. + +Any other form of ANSI sequence, such as single-character codes or alternative +initial characters, are not recognised or stripped. It would be cool to add +them though. Let me know if it would be useful for you, via the Issues on +GitHub. + + +Status & Known Problems +----------------------- + +I've personally only tested it on Windows XP (CMD, Console2), Ubuntu +(gnome-terminal, xterm), and OS X. + +Some presumably valid ANSI sequences aren't recognised (see details below), +but to my knowledge nobody has yet complained about this. Puzzling. + +See outstanding issues and wish-list: +https://github.com/tartley/colorama/issues + +If anything doesn't work for you, or doesn't do what you expected or hoped for, +I'd love to hear about it on that issues list, would be delighted by patches, +and would be happy to grant commit access to anyone who submits a working patch +or two. + + +License +------- + +Copyright Jonathan Hartley & Arnon Yaari, 2013-2020. BSD 3-Clause license; see +LICENSE file. + + +Development +----------- + +Help and fixes welcome! + +Tested on CPython 2.7, 3.5, 3.6, 3.7 and 3.8. + +No requirements other than the standard library. +Development requirements are captured in requirements-dev.txt. + +To create and populate a virtual environment:: + + ./bootstrap.ps1 # Windows + make bootstrap # Linux + +To run tests:: + + ./test.ps1 # Windows + make test # Linux + +If you use nose to run the tests, you must pass the ``-s`` flag; otherwise, +``nosetests`` applies its own proxy to ``stdout``, which confuses the unit +tests. + + +Professional support +-------------------- + +.. |tideliftlogo| image:: https://cdn2.hubspot.net/hubfs/4008838/website/logos/logos_for_download/Tidelift_primary-shorthand-logo.png + :alt: Tidelift + :target: https://tidelift.com/subscription/pkg/pypi-colorama?utm_source=pypi-colorama&utm_medium=referral&utm_campaign=readme + +.. list-table:: + :widths: 10 100 + + * - |tideliftlogo| + - Professional support for colorama is available as part of the + `Tidelift Subscription`_. + Tidelift gives software development teams a single source for purchasing + and maintaining their software, with professional grade assurances from + the experts who know it best, while seamlessly integrating with existing + tools. + +.. _Tidelift Subscription: https://tidelift.com/subscription/pkg/pypi-colorama?utm_source=pypi-colorama&utm_medium=referral&utm_campaign=readme + + +Thanks +------ + +* Marc Schlaich (schlamar) for a ``setup.py`` fix for Python2.5. +* Marc Abramowitz, reported & fixed a crash on exit with closed ``stdout``, + providing a solution to issue #7's setuptools/distutils debate, + and other fixes. +* User 'eryksun', for guidance on correctly instantiating ``ctypes.windll``. +* Matthew McCormick for politely pointing out a longstanding crash on non-Win. +* Ben Hoyt, for a magnificent fix under 64-bit Windows. +* Jesse at Empty Square for submitting a fix for examples in the README. +* User 'jamessp', an observant documentation fix for cursor positioning. +* User 'vaal1239', Dave Mckee & Lackner Kristof for a tiny but much-needed Win7 + fix. +* Julien Stuyck, for wisely suggesting Python3 compatible updates to README. +* Daniel Griffith for multiple fabulous patches. +* Oscar Lesta for a valuable fix to stop ANSI chars being sent to non-tty + output. +* Roger Binns, for many suggestions, valuable feedback, & bug reports. +* Tim Golden for thought and much appreciated feedback on the initial idea. +* User 'Zearin' for updates to the README file. +* John Szakmeister for adding support for light colors +* Charles Merriam for adding documentation to demos +* Jurko for a fix on 64-bit Windows CPython2.5 w/o ctypes +* Florian Bruhin for a fix when stdout or stderr are None +* Thomas Weininger for fixing ValueError on Windows +* Remi Rampin for better Github integration and fixes to the README file +* Simeon Visser for closing a file handle using 'with' and updating classifiers + to include Python 3.3 and 3.4 +* Andy Neff for fixing RESET of LIGHT_EX colors. +* Jonathan Hartley for the initial idea and implementation. + + + diff --git a/WebCrawler/venv/Lib/site-packages/colorama-0.4.4.dist-info/RECORD b/WebCrawler/venv/Lib/site-packages/colorama-0.4.4.dist-info/RECORD new file mode 100644 index 0000000..3516c65 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/colorama-0.4.4.dist-info/RECORD @@ -0,0 +1,18 @@ +colorama-0.4.4.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4 +colorama-0.4.4.dist-info/LICENSE.txt,sha256=ysNcAmhuXQSlpxQL-zs25zrtSWZW6JEQLkKIhteTAxg,1491 +colorama-0.4.4.dist-info/METADATA,sha256=JmU7ePpEh1xcqZV0JKcrrlU7cp5o4InDlHJXbo_FTQw,14551 +colorama-0.4.4.dist-info/RECORD,, +colorama-0.4.4.dist-info/WHEEL,sha256=gxPaqcqKPLUXaSAKwmfHO7_iAOlVvmp33DewnUluBB8,116 +colorama-0.4.4.dist-info/top_level.txt,sha256=_Kx6-Cni2BT1PEATPhrSRxo0d7kSgfBbHf5o7IF1ABw,9 +colorama/__init__.py,sha256=pCdErryzLSzDW5P-rRPBlPLqbBtIRNJB6cMgoeJns5k,239 +colorama/__pycache__/__init__.cpython-39.pyc,, +colorama/__pycache__/ansi.cpython-39.pyc,, +colorama/__pycache__/ansitowin32.cpython-39.pyc,, +colorama/__pycache__/initialise.cpython-39.pyc,, +colorama/__pycache__/win32.cpython-39.pyc,, +colorama/__pycache__/winterm.cpython-39.pyc,, +colorama/ansi.py,sha256=Top4EeEuaQdBWdteKMEcGOTeKeF19Q-Wo_6_Cj5kOzQ,2522 +colorama/ansitowin32.py,sha256=yV7CEmCb19MjnJKODZEEvMH_fnbJhwnpzo4sxZuGXmA,10517 +colorama/initialise.py,sha256=PprovDNxMTrvoNHFcL2NZjpH2XzDc8BLxLxiErfUl4k,1915 +colorama/win32.py,sha256=bJ8Il9jwaBN5BJ8bmN6FoYZ1QYuMKv2j8fGrXh7TJjw,5404 +colorama/winterm.py,sha256=2y_2b7Zsv34feAsP67mLOVc-Bgq51mdYGo571VprlrM,6438 diff --git a/WebCrawler/venv/Lib/site-packages/colorama-0.4.4.dist-info/WHEEL b/WebCrawler/venv/Lib/site-packages/colorama-0.4.4.dist-info/WHEEL new file mode 100644 index 0000000..6d38aa0 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/colorama-0.4.4.dist-info/WHEEL @@ -0,0 +1,6 @@ +Wheel-Version: 1.0 +Generator: bdist_wheel (0.35.1) +Root-Is-Purelib: true +Tag: py2-none-any +Tag: py3-none-any + diff --git a/WebCrawler/venv/Lib/site-packages/colorama-0.4.4.dist-info/top_level.txt b/WebCrawler/venv/Lib/site-packages/colorama-0.4.4.dist-info/top_level.txt new file mode 100644 index 0000000..3fcfb51 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/colorama-0.4.4.dist-info/top_level.txt @@ -0,0 +1 @@ +colorama diff --git a/WebCrawler/venv/Lib/site-packages/colorama/__init__.py b/WebCrawler/venv/Lib/site-packages/colorama/__init__.py new file mode 100644 index 0000000..b149ed7 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/colorama/__init__.py @@ -0,0 +1,6 @@ +# Copyright Jonathan Hartley 2013. BSD 3-Clause license, see LICENSE file. +from .initialise import init, deinit, reinit, colorama_text +from .ansi import Fore, Back, Style, Cursor +from .ansitowin32 import AnsiToWin32 + +__version__ = '0.4.4' diff --git a/WebCrawler/venv/Lib/site-packages/colorama/__pycache__/__init__.cpython-38.pyc b/WebCrawler/venv/Lib/site-packages/colorama/__pycache__/__init__.cpython-38.pyc new file mode 100644 index 0000000..d3cbc21 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/colorama/__pycache__/__init__.cpython-38.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/colorama/__pycache__/__init__.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/colorama/__pycache__/__init__.cpython-39.pyc new file mode 100644 index 0000000..11dea50 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/colorama/__pycache__/__init__.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/colorama/__pycache__/ansi.cpython-38.pyc b/WebCrawler/venv/Lib/site-packages/colorama/__pycache__/ansi.cpython-38.pyc new file mode 100644 index 0000000..afdbc45 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/colorama/__pycache__/ansi.cpython-38.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/colorama/__pycache__/ansi.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/colorama/__pycache__/ansi.cpython-39.pyc new file mode 100644 index 0000000..bedc68b Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/colorama/__pycache__/ansi.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/colorama/__pycache__/ansitowin32.cpython-38.pyc b/WebCrawler/venv/Lib/site-packages/colorama/__pycache__/ansitowin32.cpython-38.pyc new file mode 100644 index 0000000..856c772 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/colorama/__pycache__/ansitowin32.cpython-38.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/colorama/__pycache__/ansitowin32.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/colorama/__pycache__/ansitowin32.cpython-39.pyc new file mode 100644 index 0000000..b0c161e Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/colorama/__pycache__/ansitowin32.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/colorama/__pycache__/initialise.cpython-38.pyc b/WebCrawler/venv/Lib/site-packages/colorama/__pycache__/initialise.cpython-38.pyc new file mode 100644 index 0000000..c4494b3 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/colorama/__pycache__/initialise.cpython-38.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/colorama/__pycache__/initialise.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/colorama/__pycache__/initialise.cpython-39.pyc new file mode 100644 index 0000000..f977f7f Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/colorama/__pycache__/initialise.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/colorama/__pycache__/win32.cpython-38.pyc b/WebCrawler/venv/Lib/site-packages/colorama/__pycache__/win32.cpython-38.pyc new file mode 100644 index 0000000..ca0c015 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/colorama/__pycache__/win32.cpython-38.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/colorama/__pycache__/win32.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/colorama/__pycache__/win32.cpython-39.pyc new file mode 100644 index 0000000..8334073 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/colorama/__pycache__/win32.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/colorama/__pycache__/winterm.cpython-38.pyc b/WebCrawler/venv/Lib/site-packages/colorama/__pycache__/winterm.cpython-38.pyc new file mode 100644 index 0000000..c995bb0 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/colorama/__pycache__/winterm.cpython-38.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/colorama/__pycache__/winterm.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/colorama/__pycache__/winterm.cpython-39.pyc new file mode 100644 index 0000000..73bb556 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/colorama/__pycache__/winterm.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/colorama/ansi.py b/WebCrawler/venv/Lib/site-packages/colorama/ansi.py new file mode 100644 index 0000000..11ec695 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/colorama/ansi.py @@ -0,0 +1,102 @@ +# Copyright Jonathan Hartley 2013. BSD 3-Clause license, see LICENSE file. +''' +This module generates ANSI character codes to printing colors to terminals. +See: http://en.wikipedia.org/wiki/ANSI_escape_code +''' + +CSI = '\033[' +OSC = '\033]' +BEL = '\a' + + +def code_to_chars(code): + return CSI + str(code) + 'm' + +def set_title(title): + return OSC + '2;' + title + BEL + +def clear_screen(mode=2): + return CSI + str(mode) + 'J' + +def clear_line(mode=2): + return CSI + str(mode) + 'K' + + +class AnsiCodes(object): + def __init__(self): + # the subclasses declare class attributes which are numbers. + # Upon instantiation we define instance attributes, which are the same + # as the class attributes but wrapped with the ANSI escape sequence + for name in dir(self): + if not name.startswith('_'): + value = getattr(self, name) + setattr(self, name, code_to_chars(value)) + + +class AnsiCursor(object): + def UP(self, n=1): + return CSI + str(n) + 'A' + def DOWN(self, n=1): + return CSI + str(n) + 'B' + def FORWARD(self, n=1): + return CSI + str(n) + 'C' + def BACK(self, n=1): + return CSI + str(n) + 'D' + def POS(self, x=1, y=1): + return CSI + str(y) + ';' + str(x) + 'H' + + +class AnsiFore(AnsiCodes): + BLACK = 30 + RED = 31 + GREEN = 32 + YELLOW = 33 + BLUE = 34 + MAGENTA = 35 + CYAN = 36 + WHITE = 37 + RESET = 39 + + # These are fairly well supported, but not part of the standard. + LIGHTBLACK_EX = 90 + LIGHTRED_EX = 91 + LIGHTGREEN_EX = 92 + LIGHTYELLOW_EX = 93 + LIGHTBLUE_EX = 94 + LIGHTMAGENTA_EX = 95 + LIGHTCYAN_EX = 96 + LIGHTWHITE_EX = 97 + + +class AnsiBack(AnsiCodes): + BLACK = 40 + RED = 41 + GREEN = 42 + YELLOW = 43 + BLUE = 44 + MAGENTA = 45 + CYAN = 46 + WHITE = 47 + RESET = 49 + + # These are fairly well supported, but not part of the standard. + LIGHTBLACK_EX = 100 + LIGHTRED_EX = 101 + LIGHTGREEN_EX = 102 + LIGHTYELLOW_EX = 103 + LIGHTBLUE_EX = 104 + LIGHTMAGENTA_EX = 105 + LIGHTCYAN_EX = 106 + LIGHTWHITE_EX = 107 + + +class AnsiStyle(AnsiCodes): + BRIGHT = 1 + DIM = 2 + NORMAL = 22 + RESET_ALL = 0 + +Fore = AnsiFore() +Back = AnsiBack() +Style = AnsiStyle() +Cursor = AnsiCursor() diff --git a/WebCrawler/venv/Lib/site-packages/colorama/ansitowin32.py b/WebCrawler/venv/Lib/site-packages/colorama/ansitowin32.py new file mode 100644 index 0000000..6039a05 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/colorama/ansitowin32.py @@ -0,0 +1,258 @@ +# Copyright Jonathan Hartley 2013. BSD 3-Clause license, see LICENSE file. +import re +import sys +import os + +from .ansi import AnsiFore, AnsiBack, AnsiStyle, Style, BEL +from .winterm import WinTerm, WinColor, WinStyle +from .win32 import windll, winapi_test + + +winterm = None +if windll is not None: + winterm = WinTerm() + + +class StreamWrapper(object): + ''' + Wraps a stream (such as stdout), acting as a transparent proxy for all + attribute access apart from method 'write()', which is delegated to our + Converter instance. + ''' + def __init__(self, wrapped, converter): + # double-underscore everything to prevent clashes with names of + # attributes on the wrapped stream object. + self.__wrapped = wrapped + self.__convertor = converter + + def __getattr__(self, name): + return getattr(self.__wrapped, name) + + def __enter__(self, *args, **kwargs): + # special method lookup bypasses __getattr__/__getattribute__, see + # https://stackoverflow.com/questions/12632894/why-doesnt-getattr-work-with-exit + # thus, contextlib magic methods are not proxied via __getattr__ + return self.__wrapped.__enter__(*args, **kwargs) + + def __exit__(self, *args, **kwargs): + return self.__wrapped.__exit__(*args, **kwargs) + + def write(self, text): + self.__convertor.write(text) + + def isatty(self): + stream = self.__wrapped + if 'PYCHARM_HOSTED' in os.environ: + if stream is not None and (stream is sys.__stdout__ or stream is sys.__stderr__): + return True + try: + stream_isatty = stream.isatty + except AttributeError: + return False + else: + return stream_isatty() + + @property + def closed(self): + stream = self.__wrapped + try: + return stream.closed + except AttributeError: + return True + + +class AnsiToWin32(object): + ''' + Implements a 'write()' method which, on Windows, will strip ANSI character + sequences from the text, and if outputting to a tty, will convert them into + win32 function calls. + ''' + ANSI_CSI_RE = re.compile('\001?\033\\[((?:\\d|;)*)([a-zA-Z])\002?') # Control Sequence Introducer + ANSI_OSC_RE = re.compile('\001?\033\\]([^\a]*)(\a)\002?') # Operating System Command + + def __init__(self, wrapped, convert=None, strip=None, autoreset=False): + # The wrapped stream (normally sys.stdout or sys.stderr) + self.wrapped = wrapped + + # should we reset colors to defaults after every .write() + self.autoreset = autoreset + + # create the proxy wrapping our output stream + self.stream = StreamWrapper(wrapped, self) + + on_windows = os.name == 'nt' + # We test if the WinAPI works, because even if we are on Windows + # we may be using a terminal that doesn't support the WinAPI + # (e.g. Cygwin Terminal). In this case it's up to the terminal + # to support the ANSI codes. + conversion_supported = on_windows and winapi_test() + + # should we strip ANSI sequences from our output? + if strip is None: + strip = conversion_supported or (not self.stream.closed and not self.stream.isatty()) + self.strip = strip + + # should we should convert ANSI sequences into win32 calls? + if convert is None: + convert = conversion_supported and not self.stream.closed and self.stream.isatty() + self.convert = convert + + # dict of ansi codes to win32 functions and parameters + self.win32_calls = self.get_win32_calls() + + # are we wrapping stderr? + self.on_stderr = self.wrapped is sys.stderr + + def should_wrap(self): + ''' + True if this class is actually needed. If false, then the output + stream will not be affected, nor will win32 calls be issued, so + wrapping stdout is not actually required. This will generally be + False on non-Windows platforms, unless optional functionality like + autoreset has been requested using kwargs to init() + ''' + return self.convert or self.strip or self.autoreset + + def get_win32_calls(self): + if self.convert and winterm: + return { + AnsiStyle.RESET_ALL: (winterm.reset_all, ), + AnsiStyle.BRIGHT: (winterm.style, WinStyle.BRIGHT), + AnsiStyle.DIM: (winterm.style, WinStyle.NORMAL), + AnsiStyle.NORMAL: (winterm.style, WinStyle.NORMAL), + AnsiFore.BLACK: (winterm.fore, WinColor.BLACK), + AnsiFore.RED: (winterm.fore, WinColor.RED), + AnsiFore.GREEN: (winterm.fore, WinColor.GREEN), + AnsiFore.YELLOW: (winterm.fore, WinColor.YELLOW), + AnsiFore.BLUE: (winterm.fore, WinColor.BLUE), + AnsiFore.MAGENTA: (winterm.fore, WinColor.MAGENTA), + AnsiFore.CYAN: (winterm.fore, WinColor.CYAN), + AnsiFore.WHITE: (winterm.fore, WinColor.GREY), + AnsiFore.RESET: (winterm.fore, ), + AnsiFore.LIGHTBLACK_EX: (winterm.fore, WinColor.BLACK, True), + AnsiFore.LIGHTRED_EX: (winterm.fore, WinColor.RED, True), + AnsiFore.LIGHTGREEN_EX: (winterm.fore, WinColor.GREEN, True), + AnsiFore.LIGHTYELLOW_EX: (winterm.fore, WinColor.YELLOW, True), + AnsiFore.LIGHTBLUE_EX: (winterm.fore, WinColor.BLUE, True), + AnsiFore.LIGHTMAGENTA_EX: (winterm.fore, WinColor.MAGENTA, True), + AnsiFore.LIGHTCYAN_EX: (winterm.fore, WinColor.CYAN, True), + AnsiFore.LIGHTWHITE_EX: (winterm.fore, WinColor.GREY, True), + AnsiBack.BLACK: (winterm.back, WinColor.BLACK), + AnsiBack.RED: (winterm.back, WinColor.RED), + AnsiBack.GREEN: (winterm.back, WinColor.GREEN), + AnsiBack.YELLOW: (winterm.back, WinColor.YELLOW), + AnsiBack.BLUE: (winterm.back, WinColor.BLUE), + AnsiBack.MAGENTA: (winterm.back, WinColor.MAGENTA), + AnsiBack.CYAN: (winterm.back, WinColor.CYAN), + AnsiBack.WHITE: (winterm.back, WinColor.GREY), + AnsiBack.RESET: (winterm.back, ), + AnsiBack.LIGHTBLACK_EX: (winterm.back, WinColor.BLACK, True), + AnsiBack.LIGHTRED_EX: (winterm.back, WinColor.RED, True), + AnsiBack.LIGHTGREEN_EX: (winterm.back, WinColor.GREEN, True), + AnsiBack.LIGHTYELLOW_EX: (winterm.back, WinColor.YELLOW, True), + AnsiBack.LIGHTBLUE_EX: (winterm.back, WinColor.BLUE, True), + AnsiBack.LIGHTMAGENTA_EX: (winterm.back, WinColor.MAGENTA, True), + AnsiBack.LIGHTCYAN_EX: (winterm.back, WinColor.CYAN, True), + AnsiBack.LIGHTWHITE_EX: (winterm.back, WinColor.GREY, True), + } + return dict() + + def write(self, text): + if self.strip or self.convert: + self.write_and_convert(text) + else: + self.wrapped.write(text) + self.wrapped.flush() + if self.autoreset: + self.reset_all() + + + def reset_all(self): + if self.convert: + self.call_win32('m', (0,)) + elif not self.strip and not self.stream.closed: + self.wrapped.write(Style.RESET_ALL) + + + def write_and_convert(self, text): + ''' + Write the given text to our wrapped stream, stripping any ANSI + sequences from the text, and optionally converting them into win32 + calls. + ''' + cursor = 0 + text = self.convert_osc(text) + for match in self.ANSI_CSI_RE.finditer(text): + start, end = match.span() + self.write_plain_text(text, cursor, start) + self.convert_ansi(*match.groups()) + cursor = end + self.write_plain_text(text, cursor, len(text)) + + + def write_plain_text(self, text, start, end): + if start < end: + self.wrapped.write(text[start:end]) + self.wrapped.flush() + + + def convert_ansi(self, paramstring, command): + if self.convert: + params = self.extract_params(command, paramstring) + self.call_win32(command, params) + + + def extract_params(self, command, paramstring): + if command in 'Hf': + params = tuple(int(p) if len(p) != 0 else 1 for p in paramstring.split(';')) + while len(params) < 2: + # defaults: + params = params + (1,) + else: + params = tuple(int(p) for p in paramstring.split(';') if len(p) != 0) + if len(params) == 0: + # defaults: + if command in 'JKm': + params = (0,) + elif command in 'ABCD': + params = (1,) + + return params + + + def call_win32(self, command, params): + if command == 'm': + for param in params: + if param in self.win32_calls: + func_args = self.win32_calls[param] + func = func_args[0] + args = func_args[1:] + kwargs = dict(on_stderr=self.on_stderr) + func(*args, **kwargs) + elif command in 'J': + winterm.erase_screen(params[0], on_stderr=self.on_stderr) + elif command in 'K': + winterm.erase_line(params[0], on_stderr=self.on_stderr) + elif command in 'Hf': # cursor position - absolute + winterm.set_cursor_position(params, on_stderr=self.on_stderr) + elif command in 'ABCD': # cursor position - relative + n = params[0] + # A - up, B - down, C - forward, D - back + x, y = {'A': (0, -n), 'B': (0, n), 'C': (n, 0), 'D': (-n, 0)}[command] + winterm.cursor_adjust(x, y, on_stderr=self.on_stderr) + + + def convert_osc(self, text): + for match in self.ANSI_OSC_RE.finditer(text): + start, end = match.span() + text = text[:start] + text[end:] + paramstring, command = match.groups() + if command == BEL: + if paramstring.count(";") == 1: + params = paramstring.split(";") + # 0 - change title and icon (we will only change title) + # 1 - change icon (we don't support this) + # 2 - change title + if params[0] in '02': + winterm.set_title(params[1]) + return text diff --git a/WebCrawler/venv/Lib/site-packages/colorama/initialise.py b/WebCrawler/venv/Lib/site-packages/colorama/initialise.py new file mode 100644 index 0000000..430d066 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/colorama/initialise.py @@ -0,0 +1,80 @@ +# Copyright Jonathan Hartley 2013. BSD 3-Clause license, see LICENSE file. +import atexit +import contextlib +import sys + +from .ansitowin32 import AnsiToWin32 + + +orig_stdout = None +orig_stderr = None + +wrapped_stdout = None +wrapped_stderr = None + +atexit_done = False + + +def reset_all(): + if AnsiToWin32 is not None: # Issue #74: objects might become None at exit + AnsiToWin32(orig_stdout).reset_all() + + +def init(autoreset=False, convert=None, strip=None, wrap=True): + + if not wrap and any([autoreset, convert, strip]): + raise ValueError('wrap=False conflicts with any other arg=True') + + global wrapped_stdout, wrapped_stderr + global orig_stdout, orig_stderr + + orig_stdout = sys.stdout + orig_stderr = sys.stderr + + if sys.stdout is None: + wrapped_stdout = None + else: + sys.stdout = wrapped_stdout = \ + wrap_stream(orig_stdout, convert, strip, autoreset, wrap) + if sys.stderr is None: + wrapped_stderr = None + else: + sys.stderr = wrapped_stderr = \ + wrap_stream(orig_stderr, convert, strip, autoreset, wrap) + + global atexit_done + if not atexit_done: + atexit.register(reset_all) + atexit_done = True + + +def deinit(): + if orig_stdout is not None: + sys.stdout = orig_stdout + if orig_stderr is not None: + sys.stderr = orig_stderr + + +@contextlib.contextmanager +def colorama_text(*args, **kwargs): + init(*args, **kwargs) + try: + yield + finally: + deinit() + + +def reinit(): + if wrapped_stdout is not None: + sys.stdout = wrapped_stdout + if wrapped_stderr is not None: + sys.stderr = wrapped_stderr + + +def wrap_stream(stream, convert, strip, autoreset, wrap): + if wrap: + wrapper = AnsiToWin32(stream, + convert=convert, strip=strip, autoreset=autoreset) + if wrapper.should_wrap(): + stream = wrapper.stream + return stream diff --git a/WebCrawler/venv/Lib/site-packages/colorama/win32.py b/WebCrawler/venv/Lib/site-packages/colorama/win32.py new file mode 100644 index 0000000..c2d8360 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/colorama/win32.py @@ -0,0 +1,152 @@ +# Copyright Jonathan Hartley 2013. BSD 3-Clause license, see LICENSE file. + +# from winbase.h +STDOUT = -11 +STDERR = -12 + +try: + import ctypes + from ctypes import LibraryLoader + windll = LibraryLoader(ctypes.WinDLL) + from ctypes import wintypes +except (AttributeError, ImportError): + windll = None + SetConsoleTextAttribute = lambda *_: None + winapi_test = lambda *_: None +else: + from ctypes import byref, Structure, c_char, POINTER + + COORD = wintypes._COORD + + class CONSOLE_SCREEN_BUFFER_INFO(Structure): + """struct in wincon.h.""" + _fields_ = [ + ("dwSize", COORD), + ("dwCursorPosition", COORD), + ("wAttributes", wintypes.WORD), + ("srWindow", wintypes.SMALL_RECT), + ("dwMaximumWindowSize", COORD), + ] + def __str__(self): + return '(%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d)' % ( + self.dwSize.Y, self.dwSize.X + , self.dwCursorPosition.Y, self.dwCursorPosition.X + , self.wAttributes + , self.srWindow.Top, self.srWindow.Left, self.srWindow.Bottom, self.srWindow.Right + , self.dwMaximumWindowSize.Y, self.dwMaximumWindowSize.X + ) + + _GetStdHandle = windll.kernel32.GetStdHandle + _GetStdHandle.argtypes = [ + wintypes.DWORD, + ] + _GetStdHandle.restype = wintypes.HANDLE + + _GetConsoleScreenBufferInfo = windll.kernel32.GetConsoleScreenBufferInfo + _GetConsoleScreenBufferInfo.argtypes = [ + wintypes.HANDLE, + POINTER(CONSOLE_SCREEN_BUFFER_INFO), + ] + _GetConsoleScreenBufferInfo.restype = wintypes.BOOL + + _SetConsoleTextAttribute = windll.kernel32.SetConsoleTextAttribute + _SetConsoleTextAttribute.argtypes = [ + wintypes.HANDLE, + wintypes.WORD, + ] + _SetConsoleTextAttribute.restype = wintypes.BOOL + + _SetConsoleCursorPosition = windll.kernel32.SetConsoleCursorPosition + _SetConsoleCursorPosition.argtypes = [ + wintypes.HANDLE, + COORD, + ] + _SetConsoleCursorPosition.restype = wintypes.BOOL + + _FillConsoleOutputCharacterA = windll.kernel32.FillConsoleOutputCharacterA + _FillConsoleOutputCharacterA.argtypes = [ + wintypes.HANDLE, + c_char, + wintypes.DWORD, + COORD, + POINTER(wintypes.DWORD), + ] + _FillConsoleOutputCharacterA.restype = wintypes.BOOL + + _FillConsoleOutputAttribute = windll.kernel32.FillConsoleOutputAttribute + _FillConsoleOutputAttribute.argtypes = [ + wintypes.HANDLE, + wintypes.WORD, + wintypes.DWORD, + COORD, + POINTER(wintypes.DWORD), + ] + _FillConsoleOutputAttribute.restype = wintypes.BOOL + + _SetConsoleTitleW = windll.kernel32.SetConsoleTitleW + _SetConsoleTitleW.argtypes = [ + wintypes.LPCWSTR + ] + _SetConsoleTitleW.restype = wintypes.BOOL + + def _winapi_test(handle): + csbi = CONSOLE_SCREEN_BUFFER_INFO() + success = _GetConsoleScreenBufferInfo( + handle, byref(csbi)) + return bool(success) + + def winapi_test(): + return any(_winapi_test(h) for h in + (_GetStdHandle(STDOUT), _GetStdHandle(STDERR))) + + def GetConsoleScreenBufferInfo(stream_id=STDOUT): + handle = _GetStdHandle(stream_id) + csbi = CONSOLE_SCREEN_BUFFER_INFO() + success = _GetConsoleScreenBufferInfo( + handle, byref(csbi)) + return csbi + + def SetConsoleTextAttribute(stream_id, attrs): + handle = _GetStdHandle(stream_id) + return _SetConsoleTextAttribute(handle, attrs) + + def SetConsoleCursorPosition(stream_id, position, adjust=True): + position = COORD(*position) + # If the position is out of range, do nothing. + if position.Y <= 0 or position.X <= 0: + return + # Adjust for Windows' SetConsoleCursorPosition: + # 1. being 0-based, while ANSI is 1-based. + # 2. expecting (x,y), while ANSI uses (y,x). + adjusted_position = COORD(position.Y - 1, position.X - 1) + if adjust: + # Adjust for viewport's scroll position + sr = GetConsoleScreenBufferInfo(STDOUT).srWindow + adjusted_position.Y += sr.Top + adjusted_position.X += sr.Left + # Resume normal processing + handle = _GetStdHandle(stream_id) + return _SetConsoleCursorPosition(handle, adjusted_position) + + def FillConsoleOutputCharacter(stream_id, char, length, start): + handle = _GetStdHandle(stream_id) + char = c_char(char.encode()) + length = wintypes.DWORD(length) + num_written = wintypes.DWORD(0) + # Note that this is hard-coded for ANSI (vs wide) bytes. + success = _FillConsoleOutputCharacterA( + handle, char, length, start, byref(num_written)) + return num_written.value + + def FillConsoleOutputAttribute(stream_id, attr, length, start): + ''' FillConsoleOutputAttribute( hConsole, csbi.wAttributes, dwConSize, coordScreen, &cCharsWritten )''' + handle = _GetStdHandle(stream_id) + attribute = wintypes.WORD(attr) + length = wintypes.DWORD(length) + num_written = wintypes.DWORD(0) + # Note that this is hard-coded for ANSI (vs wide) bytes. + return _FillConsoleOutputAttribute( + handle, attribute, length, start, byref(num_written)) + + def SetConsoleTitle(title): + return _SetConsoleTitleW(title) diff --git a/WebCrawler/venv/Lib/site-packages/colorama/winterm.py b/WebCrawler/venv/Lib/site-packages/colorama/winterm.py new file mode 100644 index 0000000..0fdb4ec --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/colorama/winterm.py @@ -0,0 +1,169 @@ +# Copyright Jonathan Hartley 2013. BSD 3-Clause license, see LICENSE file. +from . import win32 + + +# from wincon.h +class WinColor(object): + BLACK = 0 + BLUE = 1 + GREEN = 2 + CYAN = 3 + RED = 4 + MAGENTA = 5 + YELLOW = 6 + GREY = 7 + +# from wincon.h +class WinStyle(object): + NORMAL = 0x00 # dim text, dim background + BRIGHT = 0x08 # bright text, dim background + BRIGHT_BACKGROUND = 0x80 # dim text, bright background + +class WinTerm(object): + + def __init__(self): + self._default = win32.GetConsoleScreenBufferInfo(win32.STDOUT).wAttributes + self.set_attrs(self._default) + self._default_fore = self._fore + self._default_back = self._back + self._default_style = self._style + # In order to emulate LIGHT_EX in windows, we borrow the BRIGHT style. + # So that LIGHT_EX colors and BRIGHT style do not clobber each other, + # we track them separately, since LIGHT_EX is overwritten by Fore/Back + # and BRIGHT is overwritten by Style codes. + self._light = 0 + + def get_attrs(self): + return self._fore + self._back * 16 + (self._style | self._light) + + def set_attrs(self, value): + self._fore = value & 7 + self._back = (value >> 4) & 7 + self._style = value & (WinStyle.BRIGHT | WinStyle.BRIGHT_BACKGROUND) + + def reset_all(self, on_stderr=None): + self.set_attrs(self._default) + self.set_console(attrs=self._default) + self._light = 0 + + def fore(self, fore=None, light=False, on_stderr=False): + if fore is None: + fore = self._default_fore + self._fore = fore + # Emulate LIGHT_EX with BRIGHT Style + if light: + self._light |= WinStyle.BRIGHT + else: + self._light &= ~WinStyle.BRIGHT + self.set_console(on_stderr=on_stderr) + + def back(self, back=None, light=False, on_stderr=False): + if back is None: + back = self._default_back + self._back = back + # Emulate LIGHT_EX with BRIGHT_BACKGROUND Style + if light: + self._light |= WinStyle.BRIGHT_BACKGROUND + else: + self._light &= ~WinStyle.BRIGHT_BACKGROUND + self.set_console(on_stderr=on_stderr) + + def style(self, style=None, on_stderr=False): + if style is None: + style = self._default_style + self._style = style + self.set_console(on_stderr=on_stderr) + + def set_console(self, attrs=None, on_stderr=False): + if attrs is None: + attrs = self.get_attrs() + handle = win32.STDOUT + if on_stderr: + handle = win32.STDERR + win32.SetConsoleTextAttribute(handle, attrs) + + def get_position(self, handle): + position = win32.GetConsoleScreenBufferInfo(handle).dwCursorPosition + # Because Windows coordinates are 0-based, + # and win32.SetConsoleCursorPosition expects 1-based. + position.X += 1 + position.Y += 1 + return position + + def set_cursor_position(self, position=None, on_stderr=False): + if position is None: + # I'm not currently tracking the position, so there is no default. + # position = self.get_position() + return + handle = win32.STDOUT + if on_stderr: + handle = win32.STDERR + win32.SetConsoleCursorPosition(handle, position) + + def cursor_adjust(self, x, y, on_stderr=False): + handle = win32.STDOUT + if on_stderr: + handle = win32.STDERR + position = self.get_position(handle) + adjusted_position = (position.Y + y, position.X + x) + win32.SetConsoleCursorPosition(handle, adjusted_position, adjust=False) + + def erase_screen(self, mode=0, on_stderr=False): + # 0 should clear from the cursor to the end of the screen. + # 1 should clear from the cursor to the beginning of the screen. + # 2 should clear the entire screen, and move cursor to (1,1) + handle = win32.STDOUT + if on_stderr: + handle = win32.STDERR + csbi = win32.GetConsoleScreenBufferInfo(handle) + # get the number of character cells in the current buffer + cells_in_screen = csbi.dwSize.X * csbi.dwSize.Y + # get number of character cells before current cursor position + cells_before_cursor = csbi.dwSize.X * csbi.dwCursorPosition.Y + csbi.dwCursorPosition.X + if mode == 0: + from_coord = csbi.dwCursorPosition + cells_to_erase = cells_in_screen - cells_before_cursor + elif mode == 1: + from_coord = win32.COORD(0, 0) + cells_to_erase = cells_before_cursor + elif mode == 2: + from_coord = win32.COORD(0, 0) + cells_to_erase = cells_in_screen + else: + # invalid mode + return + # fill the entire screen with blanks + win32.FillConsoleOutputCharacter(handle, ' ', cells_to_erase, from_coord) + # now set the buffer's attributes accordingly + win32.FillConsoleOutputAttribute(handle, self.get_attrs(), cells_to_erase, from_coord) + if mode == 2: + # put the cursor where needed + win32.SetConsoleCursorPosition(handle, (1, 1)) + + def erase_line(self, mode=0, on_stderr=False): + # 0 should clear from the cursor to the end of the line. + # 1 should clear from the cursor to the beginning of the line. + # 2 should clear the entire line. + handle = win32.STDOUT + if on_stderr: + handle = win32.STDERR + csbi = win32.GetConsoleScreenBufferInfo(handle) + if mode == 0: + from_coord = csbi.dwCursorPosition + cells_to_erase = csbi.dwSize.X - csbi.dwCursorPosition.X + elif mode == 1: + from_coord = win32.COORD(0, csbi.dwCursorPosition.Y) + cells_to_erase = csbi.dwCursorPosition.X + elif mode == 2: + from_coord = win32.COORD(0, csbi.dwCursorPosition.Y) + cells_to_erase = csbi.dwSize.X + else: + # invalid mode + return + # fill the entire screen with blanks + win32.FillConsoleOutputCharacter(handle, ' ', cells_to_erase, from_coord) + # now set the buffer's attributes accordingly + win32.FillConsoleOutputAttribute(handle, self.get_attrs(), cells_to_erase, from_coord) + + def set_title(self, title): + win32.SetConsoleTitle(title) diff --git a/WebCrawler/venv/Lib/site-packages/constantly-15.1.0.dist-info/DESCRIPTION.rst b/WebCrawler/venv/Lib/site-packages/constantly-15.1.0.dist-info/DESCRIPTION.rst new file mode 100644 index 0000000..3ff4c87 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/constantly-15.1.0.dist-info/DESCRIPTION.rst @@ -0,0 +1,18 @@ +Constantly +========== + +A library that provides symbolic constant support. +It includes collections and constants with text, numeric, and bit flag values. +Originally ``twisted.python.constants`` from the `Twisted `_ project. + + +Tests +----- + +To run tests:: + + $ tox + +This will run tests on Python 2.7, 3.3, 3.4, and PyPy, as well as doing coverage and pyflakes checks. + + diff --git a/WebCrawler/venv/Lib/site-packages/constantly-15.1.0.dist-info/INSTALLER b/WebCrawler/venv/Lib/site-packages/constantly-15.1.0.dist-info/INSTALLER new file mode 100644 index 0000000..a1b589e --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/constantly-15.1.0.dist-info/INSTALLER @@ -0,0 +1 @@ +pip diff --git a/WebCrawler/venv/Lib/site-packages/constantly-15.1.0.dist-info/METADATA b/WebCrawler/venv/Lib/site-packages/constantly-15.1.0.dist-info/METADATA new file mode 100644 index 0000000..dd2e58d --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/constantly-15.1.0.dist-info/METADATA @@ -0,0 +1,39 @@ +Metadata-Version: 2.0 +Name: constantly +Version: 15.1.0 +Summary: Symbolic constants in Python +Home-page: https://github.com/twisted/constantly +Author: Twisted Matrix Labs Developers +Author-email: UNKNOWN +License: MIT +Keywords: constants,enum,twisted +Platform: UNKNOWN +Classifier: Intended Audience :: Developers +Classifier: License :: OSI Approved :: MIT License +Classifier: Operating System :: OS Independent +Classifier: Programming Language :: Python +Classifier: Programming Language :: Python :: 2.7 +Classifier: Programming Language :: Python :: 3.3 +Classifier: Programming Language :: Python :: 3.4 +Classifier: Programming Language :: Python :: Implementation :: CPython +Classifier: Programming Language :: Python :: Implementation :: PyPy +Classifier: Topic :: Software Development :: Libraries :: Python Modules + +Constantly +========== + +A library that provides symbolic constant support. +It includes collections and constants with text, numeric, and bit flag values. +Originally ``twisted.python.constants`` from the `Twisted `_ project. + + +Tests +----- + +To run tests:: + + $ tox + +This will run tests on Python 2.7, 3.3, 3.4, and PyPy, as well as doing coverage and pyflakes checks. + + diff --git a/WebCrawler/venv/Lib/site-packages/constantly-15.1.0.dist-info/RECORD b/WebCrawler/venv/Lib/site-packages/constantly-15.1.0.dist-info/RECORD new file mode 100644 index 0000000..0210868 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/constantly-15.1.0.dist-info/RECORD @@ -0,0 +1,14 @@ +constantly-15.1.0.dist-info/DESCRIPTION.rst,sha256=sx7cfQa9trLrI06ORsPDOgA4UkW9KdPxYfygiqt50R4,397 +constantly-15.1.0.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4 +constantly-15.1.0.dist-info/METADATA,sha256=4Rt-WjQ0S-JhhzVk__zpXfYLaC7KaVjQzW9SVaBhwoY,1214 +constantly-15.1.0.dist-info/RECORD,, +constantly-15.1.0.dist-info/WHEEL,sha256=AvR0WeTpDaxT645bl5FQxUK6NPsTls2ttpcGJg3j1Xg,110 +constantly-15.1.0.dist-info/metadata.json,sha256=y1-W-VJW79doSoMail8qNDduY5oHC2JGG1-tBZ_dA0k,923 +constantly-15.1.0.dist-info/pbr.json,sha256=of4UIk4TOAoS3Gxf7oE2PwaRBcQW2ij9ZLRX1LMhy1A,47 +constantly-15.1.0.dist-info/top_level.txt,sha256=Z3LZxRT3sV7OIcLddygWy8rRaguc4a1hlGotDs-vFIM,11 +constantly/__init__.py,sha256=K2EFx89mIF8TkuE4F2ZhqeiE9UuB9S0DpFy0SNhu6uc,517 +constantly/__pycache__/__init__.cpython-39.pyc,, +constantly/__pycache__/_constants.cpython-39.pyc,, +constantly/__pycache__/_version.cpython-39.pyc,, +constantly/_constants.py,sha256=n-QpkK73fCIyebkqp6hvUMuEe3uYeIptBtl4KahxpcI,16090 +constantly/_version.py,sha256=kp8Hoj2tpH_Xpb4CONVFfYjTpEOr3EGzNRaT_IFZ3Ao,472 diff --git a/WebCrawler/venv/Lib/site-packages/constantly-15.1.0.dist-info/WHEEL b/WebCrawler/venv/Lib/site-packages/constantly-15.1.0.dist-info/WHEEL new file mode 100644 index 0000000..9dff69d --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/constantly-15.1.0.dist-info/WHEEL @@ -0,0 +1,6 @@ +Wheel-Version: 1.0 +Generator: bdist_wheel (0.24.0) +Root-Is-Purelib: true +Tag: py2-none-any +Tag: py3-none-any + diff --git a/WebCrawler/venv/Lib/site-packages/constantly-15.1.0.dist-info/metadata.json b/WebCrawler/venv/Lib/site-packages/constantly-15.1.0.dist-info/metadata.json new file mode 100644 index 0000000..6b522cd --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/constantly-15.1.0.dist-info/metadata.json @@ -0,0 +1 @@ +{"license": "MIT", "name": "constantly", "metadata_version": "2.0", "generator": "bdist_wheel (0.24.0)", "summary": "Symbolic constants in Python", "version": "15.1.0", "extensions": {"python.details": {"project_urls": {"Home": "https://github.com/twisted/constantly"}, "document_names": {"description": "DESCRIPTION.rst"}, "contacts": [{"role": "author", "name": "Twisted Matrix Labs Developers"}]}}, "keywords": ["constants", "enum", "twisted"], "classifiers": ["Intended Audience :: Developers", "License :: OSI Approved :: MIT License", "Operating System :: OS Independent", "Programming Language :: Python", "Programming Language :: Python :: 2.7", "Programming Language :: Python :: 3.3", "Programming Language :: Python :: 3.4", "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy", "Topic :: Software Development :: Libraries :: Python Modules"]} \ No newline at end of file diff --git a/WebCrawler/venv/Lib/site-packages/constantly-15.1.0.dist-info/pbr.json b/WebCrawler/venv/Lib/site-packages/constantly-15.1.0.dist-info/pbr.json new file mode 100644 index 0000000..37fc140 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/constantly-15.1.0.dist-info/pbr.json @@ -0,0 +1 @@ +{"is_release": false, "git_version": "c8375a7"} \ No newline at end of file diff --git a/WebCrawler/venv/Lib/site-packages/constantly-15.1.0.dist-info/top_level.txt b/WebCrawler/venv/Lib/site-packages/constantly-15.1.0.dist-info/top_level.txt new file mode 100644 index 0000000..6057189 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/constantly-15.1.0.dist-info/top_level.txt @@ -0,0 +1 @@ +constantly diff --git a/WebCrawler/venv/Lib/site-packages/constantly/__init__.py b/WebCrawler/venv/Lib/site-packages/constantly/__init__.py new file mode 100644 index 0000000..d9e6ec7 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/constantly/__init__.py @@ -0,0 +1,24 @@ +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +from constantly._constants import ( + NamedConstant, Names, ValueConstant, Values, FlagConstant, Flags +) + +from ._version import get_versions +__version__ = get_versions()['version'] +del get_versions + +__author__ = "Twisted Matrix Laboratories" +__license__ = "MIT" +__copyright__ = "Copyright 2011-2015 {0}".format(__author__) + + +__all__ = [ + 'NamedConstant', + 'ValueConstant', + 'FlagConstant', + 'Names', + 'Values', + 'Flags', +] diff --git a/WebCrawler/venv/Lib/site-packages/constantly/__pycache__/__init__.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/constantly/__pycache__/__init__.cpython-39.pyc new file mode 100644 index 0000000..8282d17 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/constantly/__pycache__/__init__.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/constantly/__pycache__/_constants.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/constantly/__pycache__/_constants.cpython-39.pyc new file mode 100644 index 0000000..1a0b0e8 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/constantly/__pycache__/_constants.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/constantly/__pycache__/_version.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/constantly/__pycache__/_version.cpython-39.pyc new file mode 100644 index 0000000..ee1dcae Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/constantly/__pycache__/_version.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/constantly/_constants.py b/WebCrawler/venv/Lib/site-packages/constantly/_constants.py new file mode 100644 index 0000000..44087b6 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/constantly/_constants.py @@ -0,0 +1,500 @@ +# -*- test-case-name: constantly.test.test_constants -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Symbolic constant support, including collections and constants with text, +numeric, and bit flag values. +""" + +from __future__ import division, absolute_import + +__all__ = [ + 'NamedConstant', 'ValueConstant', 'FlagConstant', + 'Names', 'Values', 'Flags'] + +from functools import partial +from itertools import count +from operator import and_, or_, xor + +_unspecified = object() +_constantOrder = partial(next, count()) + + +class _Constant(object): + """ + @ivar _index: A C{int} allocated from a shared counter in order to keep + track of the order in which L{_Constant}s are instantiated. + + @ivar name: A C{str} giving the name of this constant; only set once the + constant is initialized by L{_ConstantsContainer}. + + @ivar _container: The L{_ConstantsContainer} subclass this constant belongs + to; C{None} until the constant is initialized by that subclass. + """ + def __init__(self): + self._container = None + self._index = _constantOrder() + + + def __repr__(self): + """ + Return text identifying both which constant this is and which + collection it belongs to. + """ + return "<%s=%s>" % (self._container.__name__, self.name) + + + def __lt__(self, other): + """ + Implements C{<}. Order is defined by instantiation order. + + @param other: An object. + + @return: C{NotImplemented} if C{other} is not a constant belonging to + the same container as this constant, C{True} if this constant is + defined before C{other}, otherwise C{False}. + """ + if ( + not isinstance(other, self.__class__) or + not self._container == other._container + ): + return NotImplemented + return self._index < other._index + + + def __le__(self, other): + """ + Implements C{<=}. Order is defined by instantiation order. + + @param other: An object. + + @return: C{NotImplemented} if C{other} is not a constant belonging to + the same container as this constant, C{True} if this constant is + defined before or equal to C{other}, otherwise C{False}. + """ + if ( + not isinstance(other, self.__class__) or + not self._container == other._container + ): + return NotImplemented + return self is other or self._index < other._index + + + def __gt__(self, other): + """ + Implements C{>}. Order is defined by instantiation order. + + @param other: An object. + + @return: C{NotImplemented} if C{other} is not a constant belonging to + the same container as this constant, C{True} if this constant is + defined after C{other}, otherwise C{False}. + """ + if ( + not isinstance(other, self.__class__) or + not self._container == other._container + ): + return NotImplemented + return self._index > other._index + + + def __ge__(self, other): + """ + Implements C{>=}. Order is defined by instantiation order. + + @param other: An object. + + @return: C{NotImplemented} if C{other} is not a constant belonging to + the same container as this constant, C{True} if this constant is + defined after or equal to C{other}, otherwise C{False}. + """ + if ( + not isinstance(other, self.__class__) or + not self._container == other._container + ): + return NotImplemented + return self is other or self._index > other._index + + + def _realize(self, container, name, value): + """ + Complete the initialization of this L{_Constant}. + + @param container: The L{_ConstantsContainer} subclass this constant is + part of. + + @param name: The name of this constant in its container. + + @param value: The value of this constant; not used, as named constants + have no value apart from their identity. + """ + self._container = container + self.name = name + + + +class _ConstantsContainerType(type): + """ + L{_ConstantsContainerType} is a metaclass for creating constants container + classes. + """ + def __new__(self, name, bases, attributes): + """ + Create a new constants container class. + + If C{attributes} includes a value of C{None} for the C{"_constantType"} + key, the new class will not be initialized as a constants container and + it will behave as a normal class. + + @param name: The name of the container class. + @type name: L{str} + + @param bases: A tuple of the base classes for the new container class. + @type bases: L{tuple} of L{_ConstantsContainerType} instances + + @param attributes: The attributes of the new container class, including + any constants it is to contain. + @type attributes: L{dict} + """ + cls = super(_ConstantsContainerType, self).__new__( + self, name, bases, attributes) + + # Only realize constants in concrete _ConstantsContainer subclasses. + # Ignore intermediate base classes. + constantType = getattr(cls, '_constantType', None) + if constantType is None: + return cls + + constants = [] + for (name, descriptor) in attributes.items(): + if isinstance(descriptor, cls._constantType): + if descriptor._container is not None: + raise ValueError( + "Cannot use %s as the value of an attribute on %s" % ( + descriptor, cls.__name__)) + constants.append((descriptor._index, name, descriptor)) + + enumerants = {} + for (index, enumerant, descriptor) in sorted(constants): + value = cls._constantFactory(enumerant, descriptor) + descriptor._realize(cls, enumerant, value) + enumerants[enumerant] = descriptor + + # Save the dictionary which contains *only* constants (distinct from + # any other attributes the application may have given the container) + # where the class can use it later (eg for lookupByName). + cls._enumerants = enumerants + + return cls + + + +# In Python3 metaclasses are defined using a C{metaclass} keyword argument in +# the class definition. This would cause a syntax error in Python2. +# So we use L{type} to introduce an intermediate base class with the desired +# metaclass. +# See: +# * http://docs.python.org/2/library/functions.html#type +# * http://docs.python.org/3/reference/datamodel.html#customizing-class-creation +class _ConstantsContainer(_ConstantsContainerType('', (object,), {})): + """ + L{_ConstantsContainer} is a class with attributes used as symbolic + constants. It is up to subclasses to specify what kind of constants are + allowed. + + @cvar _constantType: Specified by a L{_ConstantsContainer} subclass to + specify the type of constants allowed by that subclass. + + @cvar _enumerants: A C{dict} mapping the names of constants (eg + L{NamedConstant} instances) found in the class definition to those + instances. + """ + + _constantType = None + + def __new__(cls): + """ + Classes representing constants containers are not intended to be + instantiated. + + The class object itself is used directly. + """ + raise TypeError("%s may not be instantiated." % (cls.__name__,)) + + + @classmethod + def _constantFactory(cls, name, descriptor): + """ + Construct the value for a new constant to add to this container. + + @param name: The name of the constant to create. + + @param descriptor: An instance of a L{_Constant} subclass (eg + L{NamedConstant}) which is assigned to C{name}. + + @return: L{NamedConstant} instances have no value apart from identity, + so return a meaningless dummy value. + """ + return _unspecified + + + @classmethod + def lookupByName(cls, name): + """ + Retrieve a constant by its name or raise a C{ValueError} if there is no + constant associated with that name. + + @param name: A C{str} giving the name of one of the constants defined + by C{cls}. + + @raise ValueError: If C{name} is not the name of one of the constants + defined by C{cls}. + + @return: The L{NamedConstant} associated with C{name}. + """ + if name in cls._enumerants: + return getattr(cls, name) + raise ValueError(name) + + + @classmethod + def iterconstants(cls): + """ + Iteration over a L{Names} subclass results in all of the constants it + contains. + + @return: an iterator the elements of which are the L{NamedConstant} + instances defined in the body of this L{Names} subclass. + """ + constants = cls._enumerants.values() + + return iter( + sorted(constants, key=lambda descriptor: descriptor._index)) + + + +class NamedConstant(_Constant): + """ + L{NamedConstant} defines an attribute to be a named constant within a + collection defined by a L{Names} subclass. + + L{NamedConstant} is only for use in the definition of L{Names} + subclasses. Do not instantiate L{NamedConstant} elsewhere and do not + subclass it. + """ + + + +class Names(_ConstantsContainer): + """ + A L{Names} subclass contains constants which differ only in their names and + identities. + """ + _constantType = NamedConstant + + + +class ValueConstant(_Constant): + """ + L{ValueConstant} defines an attribute to be a named constant within a + collection defined by a L{Values} subclass. + + L{ValueConstant} is only for use in the definition of L{Values} subclasses. + Do not instantiate L{ValueConstant} elsewhere and do not subclass it. + """ + def __init__(self, value): + _Constant.__init__(self) + self.value = value + + + +class Values(_ConstantsContainer): + """ + A L{Values} subclass contains constants which are associated with arbitrary + values. + """ + _constantType = ValueConstant + + @classmethod + def lookupByValue(cls, value): + """ + Retrieve a constant by its value or raise a C{ValueError} if there is + no constant associated with that value. + + @param value: The value of one of the constants defined by C{cls}. + + @raise ValueError: If C{value} is not the value of one of the constants + defined by C{cls}. + + @return: The L{ValueConstant} associated with C{value}. + """ + for constant in cls.iterconstants(): + if constant.value == value: + return constant + raise ValueError(value) + + + +def _flagOp(op, left, right): + """ + Implement a binary operator for a L{FlagConstant} instance. + + @param op: A two-argument callable implementing the binary operation. For + example, C{operator.or_}. + + @param left: The left-hand L{FlagConstant} instance. + @param right: The right-hand L{FlagConstant} instance. + + @return: A new L{FlagConstant} instance representing the result of the + operation. + """ + value = op(left.value, right.value) + names = op(left.names, right.names) + result = FlagConstant() + result._realize(left._container, names, value) + return result + + + +class FlagConstant(_Constant): + """ + L{FlagConstant} defines an attribute to be a flag constant within a + collection defined by a L{Flags} subclass. + + L{FlagConstant} is only for use in the definition of L{Flags} subclasses. + Do not instantiate L{FlagConstant} elsewhere and do not subclass it. + """ + def __init__(self, value=_unspecified): + _Constant.__init__(self) + self.value = value + + + def _realize(self, container, names, value): + """ + Complete the initialization of this L{FlagConstant}. + + This implementation differs from other C{_realize} implementations in + that a L{FlagConstant} may have several names which apply to it, due to + flags being combined with various operators. + + @param container: The L{Flags} subclass this constant is part of. + + @param names: When a single-flag value is being initialized, a C{str} + giving the name of that flag. This is the case which happens when + a L{Flags} subclass is being initialized and L{FlagConstant} + instances from its body are being realized. Otherwise, a C{set} of + C{str} giving names of all the flags set on this L{FlagConstant} + instance. This is the case when two flags are combined using C{|}, + for example. + """ + if isinstance(names, str): + name = names + names = set([names]) + elif len(names) == 1: + (name,) = names + else: + name = "{" + ",".join(sorted(names)) + "}" + _Constant._realize(self, container, name, value) + self.value = value + self.names = names + + + def __or__(self, other): + """ + Define C{|} on two L{FlagConstant} instances to create a new + L{FlagConstant} instance with all flags set in either instance set. + """ + return _flagOp(or_, self, other) + + + def __and__(self, other): + """ + Define C{&} on two L{FlagConstant} instances to create a new + L{FlagConstant} instance with only flags set in both instances set. + """ + return _flagOp(and_, self, other) + + + def __xor__(self, other): + """ + Define C{^} on two L{FlagConstant} instances to create a new + L{FlagConstant} instance with only flags set on exactly one instance + set. + """ + return _flagOp(xor, self, other) + + + def __invert__(self): + """ + Define C{~} on a L{FlagConstant} instance to create a new + L{FlagConstant} instance with all flags not set on this instance set. + """ + result = FlagConstant() + result._realize(self._container, set(), 0) + for flag in self._container.iterconstants(): + if flag.value & self.value == 0: + result |= flag + return result + + + def __iter__(self): + """ + @return: An iterator of flags set on this instance set. + """ + return (self._container.lookupByName(name) for name in self.names) + + + def __contains__(self, flag): + """ + @param flag: The flag to test for membership in this instance + set. + + @return: C{True} if C{flag} is in this instance set, else + C{False}. + """ + # Optimization for testing membership without iteration. + return bool(flag & self) + + + def __nonzero__(self): + """ + @return: C{False} if this flag's value is 0, else C{True}. + """ + return bool(self.value) + __bool__ = __nonzero__ + + + +class Flags(Values): + """ + A L{Flags} subclass contains constants which can be combined using the + common bitwise operators (C{|}, C{&}, etc) similar to a I{bitvector} from a + language like C. + """ + _constantType = FlagConstant + + _value = 1 + + @classmethod + def _constantFactory(cls, name, descriptor): + """ + For L{FlagConstant} instances with no explicitly defined value, assign + the next power of two as its value. + + @param name: The name of the constant to create. + + @param descriptor: An instance of a L{FlagConstant} which is assigned + to C{name}. + + @return: Either the value passed to the C{descriptor} constructor, or + the next power of 2 value which will be assigned to C{descriptor}, + relative to the value of the last defined L{FlagConstant}. + """ + if descriptor.value is _unspecified: + value = cls._value + cls._value <<= 1 + else: + value = descriptor.value + cls._value = value << 1 + return value diff --git a/WebCrawler/venv/Lib/site-packages/constantly/_version.py b/WebCrawler/venv/Lib/site-packages/constantly/_version.py new file mode 100644 index 0000000..2b1beb5 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/constantly/_version.py @@ -0,0 +1,21 @@ + +# This file was generated by 'versioneer.py' (0.15) from +# revision-control system data, or from the parent directory name of an +# unpacked source archive. Distribution tarballs contain a pre-generated copy +# of this file. + +import json +import sys + +version_json = ''' +{ + "dirty": false, + "error": null, + "full-revisionid": "c8375a7e3431792ea1b1b44678f3f6878d5e8c9a", + "version": "15.1.0" +} +''' # END VERSION_JSON + + +def get_versions(): + return json.loads(version_json) diff --git a/WebCrawler/venv/Lib/site-packages/cryptography-3.3.1.dist-info/AUTHORS.rst b/WebCrawler/venv/Lib/site-packages/cryptography-3.3.1.dist-info/AUTHORS.rst new file mode 100644 index 0000000..8ba7e0e --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/cryptography-3.3.1.dist-info/AUTHORS.rst @@ -0,0 +1,44 @@ +AUTHORS +======= + +PGP key fingerprints are enclosed in parentheses. + +* Alex Gaynor (E27D 4AA0 1651 72CB C5D2 AF2B 125F 5C67 DFE9 4084) +* Hynek Schlawack (C2A0 4F86 ACE2 8ADC F817 DBB7 AE25 3622 7F69 F181) +* Donald Stufft +* Laurens Van Houtven <_@lvh.io> (D9DC 4315 772F 8E91 DD22 B153 DFD1 3DF7 A8DD 569B) +* Christian Heimes +* Paul Kehrer (05FD 9FA1 6CF7 5735 0D91 A560 235A E5F1 29F9 ED98) +* Jarret Raim +* Alex Stapleton (A1C7 E50B 66DE 39ED C847 9665 8E3C 20D1 9BD9 5C4C) +* David Reid (0F83 CC87 B32F 482B C726 B58A 9FBF D8F4 DA89 6D74) +* Matthew Lefkowitz (06AB F638 E878 CD29 1264 18AB 7EC2 8125 0FBC 4A07) +* Konstantinos Koukopoulos (D6BD 52B6 8C99 A91C E2C8 934D 3300 566B 3A46 726E) +* Stephen Holsapple +* Terry Chia +* Matthew Iversen (2F04 3DCC D6E6 D5AC D262 2E0B C046 E8A8 7452 2973) +* Mohammed Attia +* Michael Hart +* Mark Adams (A18A 7DD3 283C CF2A B0CE FE0E C7A0 5E3F C972 098C) +* Gregory Haynes (6FB6 44BF 9FD0 EBA2 1CE9 471F B08F 42F9 0DC6 599F) +* Chelsea Winfree +* Steven Buss (1FB9 2EC1 CF93 DFD6 B47F F583 B1A5 6C22 290D A4C3) +* Andre Caron +* Jiangge Zhang (BBEC 782B 015F 71B1 5FF7 EACA 1A8C AA98 255F 5000) +* Major Hayden (1BF9 9264 9596 0033 698C 252B 7370 51E0 C101 1FB1) +* Phoebe Queen (10D4 7741 AB65 50F4 B264 3888 DA40 201A 072B C1FA) +* Google Inc. +* Amaury Forgeot d'Arc +* Dirkjan Ochtman (25BB BAC1 13C1 BFD5 AA59 4A4C 9F96 B929 3038 0381) +* Maximilian Hils +* Simo Sorce +* Thomas Sileo +* Fraser Tweedale +* Ofek Lev (FFB6 B92B 30B1 7848 546E 9912 972F E913 DAD5 A46E) +* Erik Daguerre +* Aviv Palivoda +* Chris Wolfe +* Jeremy Lainé +* Denis Gladkikh +* John Pacific (2CF6 0381 B5EF 29B7 D48C 2020 7BB9 71A0 E891 44D9) +* Marti Raudsepp diff --git a/WebCrawler/venv/Lib/site-packages/cryptography-3.3.1.dist-info/INSTALLER b/WebCrawler/venv/Lib/site-packages/cryptography-3.3.1.dist-info/INSTALLER new file mode 100644 index 0000000..a1b589e --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/cryptography-3.3.1.dist-info/INSTALLER @@ -0,0 +1 @@ +pip diff --git a/WebCrawler/venv/Lib/site-packages/cryptography-3.3.1.dist-info/LICENSE b/WebCrawler/venv/Lib/site-packages/cryptography-3.3.1.dist-info/LICENSE new file mode 100644 index 0000000..0707425 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/cryptography-3.3.1.dist-info/LICENSE @@ -0,0 +1,6 @@ +This software is made available under the terms of *either* of the licenses +found in LICENSE.APACHE or LICENSE.BSD. Contributions to cryptography are made +under the terms of *both* these licenses. + +The code used in the OS random engine is derived from CPython, and is licensed +under the terms of the PSF License Agreement. diff --git a/WebCrawler/venv/Lib/site-packages/cryptography-3.3.1.dist-info/LICENSE.APACHE b/WebCrawler/venv/Lib/site-packages/cryptography-3.3.1.dist-info/LICENSE.APACHE new file mode 100644 index 0000000..62589ed --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/cryptography-3.3.1.dist-info/LICENSE.APACHE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + https://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/WebCrawler/venv/Lib/site-packages/cryptography-3.3.1.dist-info/LICENSE.BSD b/WebCrawler/venv/Lib/site-packages/cryptography-3.3.1.dist-info/LICENSE.BSD new file mode 100644 index 0000000..ec1a29d --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/cryptography-3.3.1.dist-info/LICENSE.BSD @@ -0,0 +1,27 @@ +Copyright (c) Individual contributors. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + 3. Neither the name of PyCA Cryptography nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/WebCrawler/venv/Lib/site-packages/cryptography-3.3.1.dist-info/LICENSE.PSF b/WebCrawler/venv/Lib/site-packages/cryptography-3.3.1.dist-info/LICENSE.PSF new file mode 100644 index 0000000..4d3a4f5 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/cryptography-3.3.1.dist-info/LICENSE.PSF @@ -0,0 +1,41 @@ +1. This LICENSE AGREEMENT is between the Python Software Foundation ("PSF"), and + the Individual or Organization ("Licensee") accessing and otherwise using Python + 2.7.12 software in source or binary form and its associated documentation. + +2. Subject to the terms and conditions of this License Agreement, PSF hereby + grants Licensee a nonexclusive, royalty-free, world-wide license to reproduce, + analyze, test, perform and/or display publicly, prepare derivative works, + distribute, and otherwise use Python 2.7.12 alone or in any derivative + version, provided, however, that PSF's License Agreement and PSF's notice of + copyright, i.e., "Copyright © 2001-2016 Python Software Foundation; All Rights + Reserved" are retained in Python 2.7.12 alone or in any derivative version + prepared by Licensee. + +3. In the event Licensee prepares a derivative work that is based on or + incorporates Python 2.7.12 or any part thereof, and wants to make the + derivative work available to others as provided herein, then Licensee hereby + agrees to include in any such work a brief summary of the changes made to Python + 2.7.12. + +4. PSF is making Python 2.7.12 available to Licensee on an "AS IS" basis. + PSF MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR IMPLIED. BY WAY OF + EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND DISCLAIMS ANY REPRESENTATION OR + WARRANTY OF MERCHANTABILITY OR FITNESS FOR ANY PARTICULAR PURPOSE OR THAT THE + USE OF PYTHON 2.7.12 WILL NOT INFRINGE ANY THIRD PARTY RIGHTS. + +5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON 2.7.12 + FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS A RESULT OF + MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON 2.7.12, OR ANY DERIVATIVE + THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. + +6. This License Agreement will automatically terminate upon a material breach of + its terms and conditions. + +7. Nothing in this License Agreement shall be deemed to create any relationship + of agency, partnership, or joint venture between PSF and Licensee. This License + Agreement does not grant permission to use PSF trademarks or trade name in a + trademark sense to endorse or promote products or services of Licensee, or any + third party. + +8. By copying, installing or otherwise using Python 2.7.12, Licensee agrees + to be bound by the terms and conditions of this License Agreement. diff --git a/WebCrawler/venv/Lib/site-packages/cryptography-3.3.1.dist-info/METADATA b/WebCrawler/venv/Lib/site-packages/cryptography-3.3.1.dist-info/METADATA new file mode 100644 index 0000000..7e1bb27 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/cryptography-3.3.1.dist-info/METADATA @@ -0,0 +1,131 @@ +Metadata-Version: 2.1 +Name: cryptography +Version: 3.3.1 +Summary: cryptography is a package which provides cryptographic recipes and primitives to Python developers. +Home-page: https://github.com/pyca/cryptography +Author: The cryptography developers +Author-email: cryptography-dev@python.org +License: BSD or Apache License, Version 2.0 +Platform: UNKNOWN +Classifier: Development Status :: 5 - Production/Stable +Classifier: Intended Audience :: Developers +Classifier: License :: OSI Approved :: Apache Software License +Classifier: License :: OSI Approved :: BSD License +Classifier: Natural Language :: English +Classifier: Operating System :: MacOS :: MacOS X +Classifier: Operating System :: POSIX +Classifier: Operating System :: POSIX :: BSD +Classifier: Operating System :: POSIX :: Linux +Classifier: Operating System :: Microsoft :: Windows +Classifier: Programming Language :: Python +Classifier: Programming Language :: Python :: 2 +Classifier: Programming Language :: Python :: 2.7 +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3.6 +Classifier: Programming Language :: Python :: 3.7 +Classifier: Programming Language :: Python :: 3.8 +Classifier: Programming Language :: Python :: 3.9 +Classifier: Programming Language :: Python :: Implementation :: CPython +Classifier: Programming Language :: Python :: Implementation :: PyPy +Classifier: Topic :: Security :: Cryptography +Requires-Python: >=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.* +Description-Content-Type: text/x-rst +Requires-Dist: six (>=1.4.1) +Requires-Dist: cffi (>=1.12) +Requires-Dist: enum34 ; python_version < '3' +Requires-Dist: ipaddress ; python_version < '3' +Provides-Extra: docs +Requires-Dist: sphinx (!=1.8.0,!=3.1.0,!=3.1.1,>=1.6.5) ; extra == 'docs' +Requires-Dist: sphinx-rtd-theme ; extra == 'docs' +Provides-Extra: docstest +Requires-Dist: doc8 ; extra == 'docstest' +Requires-Dist: pyenchant (>=1.6.11) ; extra == 'docstest' +Requires-Dist: twine (>=1.12.0) ; extra == 'docstest' +Requires-Dist: sphinxcontrib-spelling (>=4.0.1) ; extra == 'docstest' +Provides-Extra: pep8test +Requires-Dist: black ; extra == 'pep8test' +Requires-Dist: flake8 ; extra == 'pep8test' +Requires-Dist: flake8-import-order ; extra == 'pep8test' +Requires-Dist: pep8-naming ; extra == 'pep8test' +Provides-Extra: ssh +Requires-Dist: bcrypt (>=3.1.5) ; extra == 'ssh' +Provides-Extra: test +Requires-Dist: pytest (!=3.9.0,!=3.9.1,!=3.9.2,>=3.6.0) ; extra == 'test' +Requires-Dist: pretend ; extra == 'test' +Requires-Dist: iso8601 ; extra == 'test' +Requires-Dist: pytz ; extra == 'test' +Requires-Dist: hypothesis (!=3.79.2,>=1.11.4) ; extra == 'test' + +pyca/cryptography +================= + +.. image:: https://img.shields.io/pypi/v/cryptography.svg + :target: https://pypi.org/project/cryptography/ + :alt: Latest Version + +.. image:: https://readthedocs.org/projects/cryptography/badge/?version=latest + :target: https://cryptography.io + :alt: Latest Docs + +.. image:: https://github.com/pyca/cryptography/workflows/CI/badge.svg?branch=master + :target: https://github.com/pyca/cryptography/actions?query=workflow%3ACI+branch%3Amaster + +.. image:: https://codecov.io/github/pyca/cryptography/coverage.svg?branch=master + :target: https://codecov.io/github/pyca/cryptography?branch=master + + +``cryptography`` is a package which provides cryptographic recipes and +primitives to Python developers. Our goal is for it to be your "cryptographic +standard library". It supports Python 2.7, Python 3.6+, and PyPy 5.4+. + +``cryptography`` includes both high level recipes and low level interfaces to +common cryptographic algorithms such as symmetric ciphers, message digests, and +key derivation functions. For example, to encrypt something with +``cryptography``'s high level symmetric encryption recipe: + +.. code-block:: pycon + + >>> from cryptography.fernet import Fernet + >>> # Put this somewhere safe! + >>> key = Fernet.generate_key() + >>> f = Fernet(key) + >>> token = f.encrypt(b"A really secret message. Not for prying eyes.") + >>> token + '...' + >>> f.decrypt(token) + 'A really secret message. Not for prying eyes.' + +You can find more information in the `documentation`_. + +You can install ``cryptography`` with: + +.. code-block:: console + + $ pip install cryptography + +For full details see `the installation documentation`_. + +Discussion +~~~~~~~~~~ + +If you run into bugs, you can file them in our `issue tracker`_. + +We maintain a `cryptography-dev`_ mailing list for development discussion. + +You can also join ``#cryptography-dev`` on Freenode to ask questions or get +involved. + +Security +~~~~~~~~ + +Need to report a security issue? Please consult our `security reporting`_ +documentation. + + +.. _`documentation`: https://cryptography.io/ +.. _`the installation documentation`: https://cryptography.io/en/latest/installation.html +.. _`issue tracker`: https://github.com/pyca/cryptography/issues +.. _`cryptography-dev`: https://mail.python.org/mailman/listinfo/cryptography-dev +.. _`security reporting`: https://cryptography.io/en/latest/security.html + + diff --git a/WebCrawler/venv/Lib/site-packages/cryptography-3.3.1.dist-info/RECORD b/WebCrawler/venv/Lib/site-packages/cryptography-3.3.1.dist-info/RECORD new file mode 100644 index 0000000..afb4cd5 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/cryptography-3.3.1.dist-info/RECORD @@ -0,0 +1,178 @@ +cryptography-3.3.1.dist-info/AUTHORS.rst,sha256=MoKTlP6yOmnLC_KXarHVQP0sItBk11dtZ7LzV0VhNB0,2475 +cryptography-3.3.1.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4 +cryptography-3.3.1.dist-info/LICENSE,sha256=Q9rSzHUqtyHNmp827OcPtTq3cTVR8tPYaU2OjFoG1uI,323 +cryptography-3.3.1.dist-info/LICENSE.APACHE,sha256=qsc7MUj20dcRHbyjIJn2jSbGRMaBOuHk8F9leaomY_4,11360 +cryptography-3.3.1.dist-info/LICENSE.BSD,sha256=YCxMdILeZHndLpeTzaJ15eY9dz2s0eymiSMqtwCPtPs,1532 +cryptography-3.3.1.dist-info/LICENSE.PSF,sha256=aT7ApmKzn5laTyUrA6YiKUVHDBtvEsoCkY5O_g32S58,2415 +cryptography-3.3.1.dist-info/METADATA,sha256=hbd8gd-p3G91RQDvbxAn_yeDdv0KH9tvaBQgUZyjiMA,5061 +cryptography-3.3.1.dist-info/RECORD,, +cryptography-3.3.1.dist-info/WHEEL,sha256=U7C5VWelc9W3WyclujATFeJWCHUyTpmirQ4KX0zevqY,100 +cryptography-3.3.1.dist-info/top_level.txt,sha256=rR2wh6A6juD02TBZNJqqonh8x9UP9Sa5Z9Hl1pCPCiM,31 +cryptography/__about__.py,sha256=TkNrvnDQIA99yxcgSMNyEQepmg0zRf32z06gm4cMCxI,835 +cryptography/__init__.py,sha256=lJ5HUOGCKi9r-XG4Y3qXq9dhCFv8RqwZKZgkQjQLboA,964 +cryptography/__pycache__/__about__.cpython-39.pyc,, +cryptography/__pycache__/__init__.cpython-39.pyc,, +cryptography/__pycache__/exceptions.cpython-39.pyc,, +cryptography/__pycache__/fernet.cpython-39.pyc,, +cryptography/__pycache__/utils.cpython-39.pyc,, +cryptography/exceptions.py,sha256=NPtDqIq1lsQ1Gb1BXkjsGIvbMrWMaKCaT8epiSgi010,1259 +cryptography/fernet.py,sha256=sg5RNOCKx9BrPV6wIfyXB9sDWJcw9-GPcPgN4lVmr8w,5980 +cryptography/hazmat/__init__.py,sha256=hEPNQw8dgjIPIn42qaLwXNRLCyTGNZeSvkQb57DPhbs,483 +cryptography/hazmat/__pycache__/__init__.cpython-39.pyc,, +cryptography/hazmat/__pycache__/_der.cpython-39.pyc,, +cryptography/hazmat/__pycache__/_oid.cpython-39.pyc,, +cryptography/hazmat/_der.py,sha256=NkwxQBcrR_KMAZCM3WKidXgx8CHFVU5iBnoFIrhQMQs,5205 +cryptography/hazmat/_oid.py,sha256=3L1KLxAsQJJoy15ZCl0T4I-PU-DVvzGS-ZTdS-PNy14,2432 +cryptography/hazmat/backends/__init__.py,sha256=EEhjIZgqApO7coGuybLXyaEaWIHcdg8oC0i2vxQ4RSI,616 +cryptography/hazmat/backends/__pycache__/__init__.cpython-39.pyc,, +cryptography/hazmat/backends/__pycache__/interfaces.cpython-39.pyc,, +cryptography/hazmat/backends/interfaces.py,sha256=GXySHrpGLgeTrjUgxOYtK6viaphO1dDKAOA95JFj_pM,10770 +cryptography/hazmat/backends/openssl/__init__.py,sha256=k4DMe228_hTuB2kY3Lwk62JdI3EmCd7VkV01zJm57ps,336 +cryptography/hazmat/backends/openssl/__pycache__/__init__.cpython-39.pyc,, +cryptography/hazmat/backends/openssl/__pycache__/aead.cpython-39.pyc,, +cryptography/hazmat/backends/openssl/__pycache__/backend.cpython-39.pyc,, +cryptography/hazmat/backends/openssl/__pycache__/ciphers.cpython-39.pyc,, +cryptography/hazmat/backends/openssl/__pycache__/cmac.cpython-39.pyc,, +cryptography/hazmat/backends/openssl/__pycache__/decode_asn1.cpython-39.pyc,, +cryptography/hazmat/backends/openssl/__pycache__/dh.cpython-39.pyc,, +cryptography/hazmat/backends/openssl/__pycache__/dsa.cpython-39.pyc,, +cryptography/hazmat/backends/openssl/__pycache__/ec.cpython-39.pyc,, +cryptography/hazmat/backends/openssl/__pycache__/ed25519.cpython-39.pyc,, +cryptography/hazmat/backends/openssl/__pycache__/ed448.cpython-39.pyc,, +cryptography/hazmat/backends/openssl/__pycache__/encode_asn1.cpython-39.pyc,, +cryptography/hazmat/backends/openssl/__pycache__/hashes.cpython-39.pyc,, +cryptography/hazmat/backends/openssl/__pycache__/hmac.cpython-39.pyc,, +cryptography/hazmat/backends/openssl/__pycache__/ocsp.cpython-39.pyc,, +cryptography/hazmat/backends/openssl/__pycache__/poly1305.cpython-39.pyc,, +cryptography/hazmat/backends/openssl/__pycache__/rsa.cpython-39.pyc,, +cryptography/hazmat/backends/openssl/__pycache__/utils.cpython-39.pyc,, +cryptography/hazmat/backends/openssl/__pycache__/x25519.cpython-39.pyc,, +cryptography/hazmat/backends/openssl/__pycache__/x448.cpython-39.pyc,, +cryptography/hazmat/backends/openssl/__pycache__/x509.cpython-39.pyc,, +cryptography/hazmat/backends/openssl/aead.py,sha256=ljOSkI7NXgXi9OyfHjm9J07m3EVHFNm9kfHAIogSWtc,5765 +cryptography/hazmat/backends/openssl/backend.py,sha256=CwITPFn7F3Bjxr_W6xFJcIIv-MLlQLJFdcnso8SS_U0,106372 +cryptography/hazmat/backends/openssl/ciphers.py,sha256=WrmBgZ2PCVbaMkHdfKmYUvoVZwwQ5O-1ylzstNadONc,8608 +cryptography/hazmat/backends/openssl/cmac.py,sha256=n34WXNXt-r0trp207u0cSKwGMth8qEiEs2jjgmHNtWE,2855 +cryptography/hazmat/backends/openssl/decode_asn1.py,sha256=BS2Y-4ZudWl-CB_fZ0YqVYIOQrnv7ziOhjpo-QIq8_o,32332 +cryptography/hazmat/backends/openssl/dh.py,sha256=1fZn8one2aSla85LIe6vXbf0qoLTDS-B7tYMcrJshnY,10239 +cryptography/hazmat/backends/openssl/dsa.py,sha256=Cp1w1Z6J_PEW-Qd2RAzfC04MU9YxqYOaef57f_QVpYI,10036 +cryptography/hazmat/backends/openssl/ec.py,sha256=c3DUb_AZ215f9AaAHyOKqBoNUSd6sFbUIDMbLrbLuLA,12071 +cryptography/hazmat/backends/openssl/ed25519.py,sha256=fInLppwHZnYgwkQQ5MdsOCux_y3kfW-290EbGn-0bKE,5618 +cryptography/hazmat/backends/openssl/ed448.py,sha256=Wp7dkPjb2Tyjzguh1bHwzXItMPqJq_A9-D7zCwHqnc8,5574 +cryptography/hazmat/backends/openssl/encode_asn1.py,sha256=5tQmLfLEyKTm3Eg_GfhGJcPtuQ0Ef2OdbsgxVC2fcMc,24075 +cryptography/hazmat/backends/openssl/hashes.py,sha256=n6XJwCI-2OU6FndiatFbE_Pgb3f1NoVuHwpCMW0z340,3117 +cryptography/hazmat/backends/openssl/hmac.py,sha256=D_YcF2OiLSfrWtA7fksLiKWcaVh-G1igpqNHuM5l62c,2933 +cryptography/hazmat/backends/openssl/ocsp.py,sha256=NEGrc30GfPBLbjnt-K3K48-dZK2dEyQa2oCyv7-laMs,14028 +cryptography/hazmat/backends/openssl/poly1305.py,sha256=LiovW4SvSUhWA109IkLlw4nnokmF9m24V4pGxqoPmMI,2393 +cryptography/hazmat/backends/openssl/rsa.py,sha256=hcBFzZ51LA2UJfVGf00xul5nWLeT-9Sz7ufCKls195w,19577 +cryptography/hazmat/backends/openssl/utils.py,sha256=-JMyOgOlplSWL5zTu_3-vl5_gE1FBK3ew6n0Zs35QYo,2348 +cryptography/hazmat/backends/openssl/x25519.py,sha256=-MNAPGS_DZ37-skSn17-gIakFLoJmuNx8PlC8s2-00g,4488 +cryptography/hazmat/backends/openssl/x448.py,sha256=5WH3Rw7kZGLS3EDDVzjrYriAG-tzUnyWetyqMYTiEhA,4011 +cryptography/hazmat/backends/openssl/x509.py,sha256=EMN9qSPW1BVZ1VAOHzgi8oO8idI8iOb0wrWjdrr5FpI,21620 +cryptography/hazmat/bindings/__init__.py,sha256=0wGw2OF9R7fHX7NWENCmrsYigbXHU2ojgn-N4Rkjs9U,246 +cryptography/hazmat/bindings/__pycache__/__init__.cpython-39.pyc,, +cryptography/hazmat/bindings/_openssl.pyd,sha256=gRWv_p5t8TRUr84ge7XZwZLZOhnEQ32hIIhQZJ_typg,3107328 +cryptography/hazmat/bindings/_padding.pyd,sha256=pEz1ogYxleVnxfq36gWUqsccCe1KgSoIw_F8nkvg_SY,13824 +cryptography/hazmat/bindings/openssl/__init__.py,sha256=0wGw2OF9R7fHX7NWENCmrsYigbXHU2ojgn-N4Rkjs9U,246 +cryptography/hazmat/bindings/openssl/__pycache__/__init__.cpython-39.pyc,, +cryptography/hazmat/bindings/openssl/__pycache__/_conditional.cpython-39.pyc,, +cryptography/hazmat/bindings/openssl/__pycache__/binding.cpython-39.pyc,, +cryptography/hazmat/bindings/openssl/_conditional.py,sha256=6-EwpZeSqbLNRPhzsXFPTO498wLGaDXW-LvkqiJm4vQ,8291 +cryptography/hazmat/bindings/openssl/binding.py,sha256=yT5e2JrzANd6FG__us6aj9ocb48EnNJK61cKwrpeM08,5816 +cryptography/hazmat/primitives/__init__.py,sha256=0wGw2OF9R7fHX7NWENCmrsYigbXHU2ojgn-N4Rkjs9U,246 +cryptography/hazmat/primitives/__pycache__/__init__.cpython-39.pyc,, +cryptography/hazmat/primitives/__pycache__/cmac.cpython-39.pyc,, +cryptography/hazmat/primitives/__pycache__/constant_time.cpython-39.pyc,, +cryptography/hazmat/primitives/__pycache__/hashes.cpython-39.pyc,, +cryptography/hazmat/primitives/__pycache__/hmac.cpython-39.pyc,, +cryptography/hazmat/primitives/__pycache__/keywrap.cpython-39.pyc,, +cryptography/hazmat/primitives/__pycache__/padding.cpython-39.pyc,, +cryptography/hazmat/primitives/__pycache__/poly1305.cpython-39.pyc,, +cryptography/hazmat/primitives/asymmetric/__init__.py,sha256=WhUn3tGxoLAxGAsZHElJ2aOILXSh55AZi04MBudYmQA,1020 +cryptography/hazmat/primitives/asymmetric/__pycache__/__init__.cpython-39.pyc,, +cryptography/hazmat/primitives/asymmetric/__pycache__/dh.cpython-39.pyc,, +cryptography/hazmat/primitives/asymmetric/__pycache__/dsa.cpython-39.pyc,, +cryptography/hazmat/primitives/asymmetric/__pycache__/ec.cpython-39.pyc,, +cryptography/hazmat/primitives/asymmetric/__pycache__/ed25519.cpython-39.pyc,, +cryptography/hazmat/primitives/asymmetric/__pycache__/ed448.cpython-39.pyc,, +cryptography/hazmat/primitives/asymmetric/__pycache__/padding.cpython-39.pyc,, +cryptography/hazmat/primitives/asymmetric/__pycache__/rsa.cpython-39.pyc,, +cryptography/hazmat/primitives/asymmetric/__pycache__/utils.cpython-39.pyc,, +cryptography/hazmat/primitives/asymmetric/__pycache__/x25519.cpython-39.pyc,, +cryptography/hazmat/primitives/asymmetric/__pycache__/x448.cpython-39.pyc,, +cryptography/hazmat/primitives/asymmetric/dh.py,sha256=kuyPcccLeOYy4OuGkegEyqMSzRo-QyjlUw463jzfrGs,5859 +cryptography/hazmat/primitives/asymmetric/dsa.py,sha256=XuE2mUXl-fXi2q7w22qKyiCTFUz-852cFTwV4WOUQgw,7181 +cryptography/hazmat/primitives/asymmetric/ec.py,sha256=2rorlIEXHGkLnI8bbeFKMRr-gJfEipuJigQDQh4xk7w,14006 +cryptography/hazmat/primitives/asymmetric/ed25519.py,sha256=rfImUQH-PcTliuxiF864aSww7dQCWVwZgjPPbDXiGlI,2401 +cryptography/hazmat/primitives/asymmetric/ed448.py,sha256=JyrEHwYF_Ftj_E60t-Gmvm3CGnQSxVbasptZBW84eBk,2328 +cryptography/hazmat/primitives/asymmetric/padding.py,sha256=2pPqBu4dGERtFPHnPRTZ0iRO_XY9hr9RTwlTcr_J5bw,2250 +cryptography/hazmat/primitives/asymmetric/rsa.py,sha256=MgxdkA8PWlXGt2lMPpnV9QYYvQnYTFjb0RtJRDjnlfU,10672 +cryptography/hazmat/primitives/asymmetric/utils.py,sha256=w2lQIcKrFvS9D_Ekt7qWed39TXM6hueg72FFrfwIo58,1201 +cryptography/hazmat/primitives/asymmetric/x25519.py,sha256=vrN1jcO6sjbQrc7auIlf2aEvcH3P17cKUuaVXxaTvxI,2277 +cryptography/hazmat/primitives/asymmetric/x448.py,sha256=u3v-L1IJIG2RyLVTh7FMkXh_Y-oVb3HdEj5b1c-JlKk,2255 +cryptography/hazmat/primitives/ciphers/__init__.py,sha256=mi4yR3Fxc4-Au3yX4PyhFNaiFn0yywZKiTzecdI77EI,647 +cryptography/hazmat/primitives/ciphers/__pycache__/__init__.cpython-39.pyc,, +cryptography/hazmat/primitives/ciphers/__pycache__/aead.cpython-39.pyc,, +cryptography/hazmat/primitives/ciphers/__pycache__/algorithms.cpython-39.pyc,, +cryptography/hazmat/primitives/ciphers/__pycache__/base.cpython-39.pyc,, +cryptography/hazmat/primitives/ciphers/__pycache__/modes.cpython-39.pyc,, +cryptography/hazmat/primitives/ciphers/aead.py,sha256=lXgZOxlbxtBp1k7KmlqgiN_Xu6yPsJE_DNJLwsgm0o0,6134 +cryptography/hazmat/primitives/ciphers/algorithms.py,sha256=GKFIhvOoqsYscjjP7onl8XnAmOa-kSQ6jiMMS2zeGBM,4225 +cryptography/hazmat/primitives/ciphers/base.py,sha256=vceN5l7yxLWmNTptlzC3gmfFY-K_ANKk4HdNl2Ptz2k,7253 +cryptography/hazmat/primitives/ciphers/modes.py,sha256=-0VTtHN3kKO_Jyc_iLAgp8bqtsXJY5V2F__Bkr6nvtM,6805 +cryptography/hazmat/primitives/cmac.py,sha256=eJpysDFbc7W6OiplzWKWrL4owy30Cq6Nsao8mzapqbE,2130 +cryptography/hazmat/primitives/constant_time.py,sha256=_x4mrHW-9ihfgY89BwhATFiIuG2_1l-HMkCxmOUkydM,430 +cryptography/hazmat/primitives/hashes.py,sha256=dzL1QcEFj4eElzczo8QmuOeooZ96EFwBy3c-6cpew0w,6315 +cryptography/hazmat/primitives/hmac.py,sha256=AYzTQMDiruKmZKKLR6ceVjX5yQ3mpciWIx__tpNLyr4,2306 +cryptography/hazmat/primitives/kdf/__init__.py,sha256=nod5HjPswjZr8wFp6Tsu6en9blHYF3khgXI5R0zIcnM,771 +cryptography/hazmat/primitives/kdf/__pycache__/__init__.cpython-39.pyc,, +cryptography/hazmat/primitives/kdf/__pycache__/concatkdf.cpython-39.pyc,, +cryptography/hazmat/primitives/kdf/__pycache__/hkdf.cpython-39.pyc,, +cryptography/hazmat/primitives/kdf/__pycache__/kbkdf.cpython-39.pyc,, +cryptography/hazmat/primitives/kdf/__pycache__/pbkdf2.cpython-39.pyc,, +cryptography/hazmat/primitives/kdf/__pycache__/scrypt.cpython-39.pyc,, +cryptography/hazmat/primitives/kdf/__pycache__/x963kdf.cpython-39.pyc,, +cryptography/hazmat/primitives/kdf/concatkdf.py,sha256=gW-xAU6sPE6aZhg_G9ucZ5b_uctSbPcfSpHyyt7Q8MA,4095 +cryptography/hazmat/primitives/kdf/hkdf.py,sha256=SJJQzeQ9OH0t3tUdUq2GT6IQXv9oPLDjulT7wnLTkMg,3598 +cryptography/hazmat/primitives/kdf/kbkdf.py,sha256=awf7zessT-amokp2VBdyW8TWrDnmTXGzHHX4scBO9Uc,5100 +cryptography/hazmat/primitives/kdf/pbkdf2.py,sha256=RYexIlGomzUEU-_QQXTW81rdY5YVZB30XrfnJq8NsIU,2220 +cryptography/hazmat/primitives/kdf/scrypt.py,sha256=C0C3m-gEnlLlAVxzRFdzx1mfDuWs_BkZDoSV2hfahfk,2268 +cryptography/hazmat/primitives/kdf/x963kdf.py,sha256=26-b_ckyUYiqbWM9mZ7FEWbuvR7eTLksIeWQeW1TJ04,2407 +cryptography/hazmat/primitives/keywrap.py,sha256=fF-HA5ETz9RH8s8LB94uDoWRLPvwPkYAC5_Kylej6sA,5730 +cryptography/hazmat/primitives/padding.py,sha256=zeJmjPfX8Cx_gqO45FDBNe8iN2trPr0ULyBsz1Kmyu4,6173 +cryptography/hazmat/primitives/poly1305.py,sha256=NNC1WYiYQGNJ8mblkaHRxBm1PLdaKRzkILocsYH5zgY,1679 +cryptography/hazmat/primitives/serialization/__init__.py,sha256=eLzmqoHgVlPK1aTGiEfpaIrUf9mX5PRrM7IHEc8FeQU,1132 +cryptography/hazmat/primitives/serialization/__pycache__/__init__.cpython-39.pyc,, +cryptography/hazmat/primitives/serialization/__pycache__/base.cpython-39.pyc,, +cryptography/hazmat/primitives/serialization/__pycache__/pkcs12.cpython-39.pyc,, +cryptography/hazmat/primitives/serialization/__pycache__/pkcs7.cpython-39.pyc,, +cryptography/hazmat/primitives/serialization/__pycache__/ssh.cpython-39.pyc,, +cryptography/hazmat/primitives/serialization/base.py,sha256=ZSzV-5zl2Bt_mmihcPqieBC6UjMSryUaehgExvjZksg,2249 +cryptography/hazmat/primitives/serialization/pkcs12.py,sha256=oJxangAtSSsniXfguLaoPgejVchs-VpCTBdWSW4rF54,1853 +cryptography/hazmat/primitives/serialization/pkcs7.py,sha256=vGlw_2R4VeLWtoRxkfz8fMLE5i_CCdaY9bEtYMV62rk,4625 +cryptography/hazmat/primitives/serialization/ssh.py,sha256=a_FKWuqpHO-RzUBEoBWS5q7WyMZwS56MD92Wr6j3KBA,21682 +cryptography/hazmat/primitives/twofactor/__init__.py,sha256=BWrm3DKDoAa281E7U_nzz8v44OmAiXmlIycFcsehwfE,288 +cryptography/hazmat/primitives/twofactor/__pycache__/__init__.cpython-39.pyc,, +cryptography/hazmat/primitives/twofactor/__pycache__/hotp.cpython-39.pyc,, +cryptography/hazmat/primitives/twofactor/__pycache__/totp.cpython-39.pyc,, +cryptography/hazmat/primitives/twofactor/__pycache__/utils.cpython-39.pyc,, +cryptography/hazmat/primitives/twofactor/hotp.py,sha256=2uCTCTHMFmWL9kOjA890F0CVrljsvOjJYISKBup7GyI,2679 +cryptography/hazmat/primitives/twofactor/totp.py,sha256=iJRTxPNWPdsTQHePgSE6KGdRNURTv188VNqpyvBwvBY,1780 +cryptography/hazmat/primitives/twofactor/utils.py,sha256=ZKZSOL2cLsGCsSNfx3kYlYt91A4bcU1w9up2EL1hwaA,982 +cryptography/utils.py,sha256=QpZgLOABfeaDciPlrF-W8giJiOL2AzU6Ajjq6h6WkzY,4745 +cryptography/x509/__init__.py,sha256=1juFH-nvLS7kU0x52VMN7pN6s7H55Y86NqUszaBhhi4,7699 +cryptography/x509/__pycache__/__init__.cpython-39.pyc,, +cryptography/x509/__pycache__/base.cpython-39.pyc,, +cryptography/x509/__pycache__/certificate_transparency.cpython-39.pyc,, +cryptography/x509/__pycache__/extensions.cpython-39.pyc,, +cryptography/x509/__pycache__/general_name.cpython-39.pyc,, +cryptography/x509/__pycache__/name.cpython-39.pyc,, +cryptography/x509/__pycache__/ocsp.cpython-39.pyc,, +cryptography/x509/__pycache__/oid.cpython-39.pyc,, +cryptography/x509/base.py,sha256=burWvWUouPiPzmPUzNZUzEe64gR-WMkNyiDpjYCvEc8,26409 +cryptography/x509/certificate_transparency.py,sha256=eJ9lrITdyMn4XsrcVdrTaFVI_RR7mX_VzMZyiaEpbps,1000 +cryptography/x509/extensions.py,sha256=HOwYCKAy-4qK5eWWYB4UnJejC9Ru3FBQMsLXodasR9Y,52924 +cryptography/x509/general_name.py,sha256=nNIG--rJ-TzREkhEq727Fe3tjvxVflW7iPIMjJs6LrI,7942 +cryptography/x509/name.py,sha256=j2khdee8jQBkbZd4RV60ji8V0ZngbsB07i5cnflDBPk,8291 +cryptography/x509/ocsp.py,sha256=nr5Bk3B_b9LaG-1njEmo0f_smAg2B6CU5Wr6wMr81MI,13245 +cryptography/x509/oid.py,sha256=Wp6Y4WMrFa7vsUmV4tbMvPPAl0Iiu4QxQ7on2np94QU,12594 diff --git a/WebCrawler/venv/Lib/site-packages/cryptography-3.3.1.dist-info/WHEEL b/WebCrawler/venv/Lib/site-packages/cryptography-3.3.1.dist-info/WHEEL new file mode 100644 index 0000000..f458692 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/cryptography-3.3.1.dist-info/WHEEL @@ -0,0 +1,5 @@ +Wheel-Version: 1.0 +Generator: bdist_wheel (0.36.1) +Root-Is-Purelib: false +Tag: cp36-abi3-win_amd64 + diff --git a/WebCrawler/venv/Lib/site-packages/cryptography-3.3.1.dist-info/top_level.txt b/WebCrawler/venv/Lib/site-packages/cryptography-3.3.1.dist-info/top_level.txt new file mode 100644 index 0000000..52ccfc6 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/cryptography-3.3.1.dist-info/top_level.txt @@ -0,0 +1,3 @@ +_openssl +_padding +cryptography diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/__about__.py b/WebCrawler/venv/Lib/site-packages/cryptography/__about__.py new file mode 100644 index 0000000..0c7eaaa --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/cryptography/__about__.py @@ -0,0 +1,31 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function + +__all__ = [ + "__title__", + "__summary__", + "__uri__", + "__version__", + "__author__", + "__email__", + "__license__", + "__copyright__", +] + +__title__ = "cryptography" +__summary__ = ( + "cryptography is a package which provides cryptographic recipes" + " and primitives to Python developers." +) +__uri__ = "https://github.com/pyca/cryptography" + +__version__ = "3.3.1" + +__author__ = "The cryptography developers" +__email__ = "cryptography-dev@python.org" + +__license__ = "BSD or Apache License, Version 2.0" +__copyright__ = "Copyright 2013-2020 {}".format(__author__) diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/__init__.py b/WebCrawler/venv/Lib/site-packages/cryptography/__init__.py new file mode 100644 index 0000000..465671e --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/cryptography/__init__.py @@ -0,0 +1,41 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function + +import sys +import warnings + +from cryptography.__about__ import ( + __author__, + __copyright__, + __email__, + __license__, + __summary__, + __title__, + __uri__, + __version__, +) +from cryptography.utils import CryptographyDeprecationWarning + + +__all__ = [ + "__title__", + "__summary__", + "__uri__", + "__version__", + "__author__", + "__email__", + "__license__", + "__copyright__", +] + +if sys.version_info[0] == 2: + warnings.warn( + "Python 2 is no longer supported by the Python core team. Support for " + "it is now deprecated in cryptography, and will be removed in the " + "next release.", + CryptographyDeprecationWarning, + stacklevel=2, + ) diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/__pycache__/__about__.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/cryptography/__pycache__/__about__.cpython-39.pyc new file mode 100644 index 0000000..a67c662 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/cryptography/__pycache__/__about__.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/__pycache__/__init__.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/cryptography/__pycache__/__init__.cpython-39.pyc new file mode 100644 index 0000000..dd56db4 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/cryptography/__pycache__/__init__.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/__pycache__/exceptions.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/cryptography/__pycache__/exceptions.cpython-39.pyc new file mode 100644 index 0000000..1a2244c Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/cryptography/__pycache__/exceptions.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/__pycache__/fernet.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/cryptography/__pycache__/fernet.cpython-39.pyc new file mode 100644 index 0000000..c95d979 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/cryptography/__pycache__/fernet.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/__pycache__/utils.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/cryptography/__pycache__/utils.cpython-39.pyc new file mode 100644 index 0000000..8c8e504 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/cryptography/__pycache__/utils.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/exceptions.py b/WebCrawler/venv/Lib/site-packages/cryptography/exceptions.py new file mode 100644 index 0000000..1d52d7d --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/cryptography/exceptions.py @@ -0,0 +1,58 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function + +from enum import Enum + + +class _Reasons(Enum): + BACKEND_MISSING_INTERFACE = 0 + UNSUPPORTED_HASH = 1 + UNSUPPORTED_CIPHER = 2 + UNSUPPORTED_PADDING = 3 + UNSUPPORTED_MGF = 4 + UNSUPPORTED_PUBLIC_KEY_ALGORITHM = 5 + UNSUPPORTED_ELLIPTIC_CURVE = 6 + UNSUPPORTED_SERIALIZATION = 7 + UNSUPPORTED_X509 = 8 + UNSUPPORTED_EXCHANGE_ALGORITHM = 9 + UNSUPPORTED_DIFFIE_HELLMAN = 10 + UNSUPPORTED_MAC = 11 + + +class UnsupportedAlgorithm(Exception): + def __init__(self, message, reason=None): + super(UnsupportedAlgorithm, self).__init__(message) + self._reason = reason + + +class AlreadyFinalized(Exception): + pass + + +class AlreadyUpdated(Exception): + pass + + +class NotYetFinalized(Exception): + pass + + +class InvalidTag(Exception): + pass + + +class InvalidSignature(Exception): + pass + + +class InternalError(Exception): + def __init__(self, msg, err_code): + super(InternalError, self).__init__(msg) + self.err_code = err_code + + +class InvalidKey(Exception): + pass diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/fernet.py b/WebCrawler/venv/Lib/site-packages/cryptography/fernet.py new file mode 100644 index 0000000..00c2528 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/cryptography/fernet.py @@ -0,0 +1,190 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function + +import base64 +import binascii +import os +import struct +import time + +import six + +from cryptography import utils +from cryptography.exceptions import InvalidSignature +from cryptography.hazmat.backends import _get_backend +from cryptography.hazmat.primitives import hashes, padding +from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes +from cryptography.hazmat.primitives.hmac import HMAC + + +class InvalidToken(Exception): + pass + + +_MAX_CLOCK_SKEW = 60 + + +class Fernet(object): + def __init__(self, key, backend=None): + backend = _get_backend(backend) + + key = base64.urlsafe_b64decode(key) + if len(key) != 32: + raise ValueError( + "Fernet key must be 32 url-safe base64-encoded bytes." + ) + + self._signing_key = key[:16] + self._encryption_key = key[16:] + self._backend = backend + + @classmethod + def generate_key(cls): + return base64.urlsafe_b64encode(os.urandom(32)) + + def encrypt(self, data): + return self.encrypt_at_time(data, int(time.time())) + + def encrypt_at_time(self, data, current_time): + iv = os.urandom(16) + return self._encrypt_from_parts(data, current_time, iv) + + def _encrypt_from_parts(self, data, current_time, iv): + utils._check_bytes("data", data) + + padder = padding.PKCS7(algorithms.AES.block_size).padder() + padded_data = padder.update(data) + padder.finalize() + encryptor = Cipher( + algorithms.AES(self._encryption_key), modes.CBC(iv), self._backend + ).encryptor() + ciphertext = encryptor.update(padded_data) + encryptor.finalize() + + basic_parts = ( + b"\x80" + struct.pack(">Q", current_time) + iv + ciphertext + ) + + h = HMAC(self._signing_key, hashes.SHA256(), backend=self._backend) + h.update(basic_parts) + hmac = h.finalize() + return base64.urlsafe_b64encode(basic_parts + hmac) + + def decrypt(self, token, ttl=None): + timestamp, data = Fernet._get_unverified_token_data(token) + return self._decrypt_data(data, timestamp, ttl, int(time.time())) + + def decrypt_at_time(self, token, ttl, current_time): + if ttl is None: + raise ValueError( + "decrypt_at_time() can only be used with a non-None ttl" + ) + timestamp, data = Fernet._get_unverified_token_data(token) + return self._decrypt_data(data, timestamp, ttl, current_time) + + def extract_timestamp(self, token): + timestamp, data = Fernet._get_unverified_token_data(token) + # Verify the token was not tampered with. + self._verify_signature(data) + return timestamp + + @staticmethod + def _get_unverified_token_data(token): + utils._check_bytes("token", token) + try: + data = base64.urlsafe_b64decode(token) + except (TypeError, binascii.Error): + raise InvalidToken + + if not data or six.indexbytes(data, 0) != 0x80: + raise InvalidToken + + try: + (timestamp,) = struct.unpack(">Q", data[1:9]) + except struct.error: + raise InvalidToken + return timestamp, data + + def _verify_signature(self, data): + h = HMAC(self._signing_key, hashes.SHA256(), backend=self._backend) + h.update(data[:-32]) + try: + h.verify(data[-32:]) + except InvalidSignature: + raise InvalidToken + + def _decrypt_data(self, data, timestamp, ttl, current_time): + if ttl is not None: + if timestamp + ttl < current_time: + raise InvalidToken + + if current_time + _MAX_CLOCK_SKEW < timestamp: + raise InvalidToken + + self._verify_signature(data) + + iv = data[9:25] + ciphertext = data[25:-32] + decryptor = Cipher( + algorithms.AES(self._encryption_key), modes.CBC(iv), self._backend + ).decryptor() + plaintext_padded = decryptor.update(ciphertext) + try: + plaintext_padded += decryptor.finalize() + except ValueError: + raise InvalidToken + unpadder = padding.PKCS7(algorithms.AES.block_size).unpadder() + + unpadded = unpadder.update(plaintext_padded) + try: + unpadded += unpadder.finalize() + except ValueError: + raise InvalidToken + return unpadded + + +class MultiFernet(object): + def __init__(self, fernets): + fernets = list(fernets) + if not fernets: + raise ValueError( + "MultiFernet requires at least one Fernet instance" + ) + self._fernets = fernets + + def encrypt(self, msg): + return self.encrypt_at_time(msg, int(time.time())) + + def encrypt_at_time(self, msg, current_time): + return self._fernets[0].encrypt_at_time(msg, current_time) + + def rotate(self, msg): + timestamp, data = Fernet._get_unverified_token_data(msg) + for f in self._fernets: + try: + p = f._decrypt_data(data, timestamp, None, None) + break + except InvalidToken: + pass + else: + raise InvalidToken + + iv = os.urandom(16) + return self._fernets[0]._encrypt_from_parts(p, timestamp, iv) + + def decrypt(self, msg, ttl=None): + for f in self._fernets: + try: + return f.decrypt(msg, ttl) + except InvalidToken: + pass + raise InvalidToken + + def decrypt_at_time(self, msg, ttl, current_time): + for f in self._fernets: + try: + return f.decrypt_at_time(msg, ttl, current_time) + except InvalidToken: + pass + raise InvalidToken diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/__init__.py b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/__init__.py new file mode 100644 index 0000000..9f06a99 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/__init__.py @@ -0,0 +1,11 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. +""" +Hazardous Materials + +This is a "Hazardous Materials" module. You should ONLY use it if you're +100% absolutely sure that you know what you're doing because this module +is full of land mines, dragons, and dinosaurs with laser guns. +""" +from __future__ import absolute_import, division, print_function diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/__pycache__/__init__.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/__pycache__/__init__.cpython-39.pyc new file mode 100644 index 0000000..253b019 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/__pycache__/__init__.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/__pycache__/_der.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/__pycache__/_der.cpython-39.pyc new file mode 100644 index 0000000..5d72ad3 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/__pycache__/_der.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/__pycache__/_oid.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/__pycache__/_oid.cpython-39.pyc new file mode 100644 index 0000000..8c85f85 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/__pycache__/_oid.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/_der.py b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/_der.py new file mode 100644 index 0000000..462b911 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/_der.py @@ -0,0 +1,156 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function + +import six + +from cryptography.utils import int_from_bytes, int_to_bytes + + +# This module contains a lightweight DER encoder and decoder. See X.690 for the +# specification. This module intentionally does not implement the more complex +# BER encoding, only DER. +# +# Note this implementation treats an element's constructed bit as part of the +# tag. This is fine for DER, where the bit is always computable from the type. + + +CONSTRUCTED = 0x20 +CONTEXT_SPECIFIC = 0x80 + +INTEGER = 0x02 +BIT_STRING = 0x03 +OCTET_STRING = 0x04 +NULL = 0x05 +OBJECT_IDENTIFIER = 0x06 +SEQUENCE = 0x10 | CONSTRUCTED +SET = 0x11 | CONSTRUCTED +PRINTABLE_STRING = 0x13 +UTC_TIME = 0x17 +GENERALIZED_TIME = 0x18 + + +class DERReader(object): + def __init__(self, data): + self.data = memoryview(data) + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, tb): + if exc_value is None: + self.check_empty() + + def is_empty(self): + return len(self.data) == 0 + + def check_empty(self): + if not self.is_empty(): + raise ValueError("Invalid DER input: trailing data") + + def read_byte(self): + if len(self.data) < 1: + raise ValueError("Invalid DER input: insufficient data") + ret = six.indexbytes(self.data, 0) + self.data = self.data[1:] + return ret + + def read_bytes(self, n): + if len(self.data) < n: + raise ValueError("Invalid DER input: insufficient data") + ret = self.data[:n] + self.data = self.data[n:] + return ret + + def read_any_element(self): + tag = self.read_byte() + # Tag numbers 31 or higher are stored in multiple bytes. No supported + # ASN.1 types use such tags, so reject these. + if tag & 0x1F == 0x1F: + raise ValueError("Invalid DER input: unexpected high tag number") + length_byte = self.read_byte() + if length_byte & 0x80 == 0: + # If the high bit is clear, the first length byte is the length. + length = length_byte + else: + # If the high bit is set, the first length byte encodes the length + # of the length. + length_byte &= 0x7F + if length_byte == 0: + raise ValueError( + "Invalid DER input: indefinite length form is not allowed " + "in DER" + ) + length = 0 + for i in range(length_byte): + length <<= 8 + length |= self.read_byte() + if length == 0: + raise ValueError( + "Invalid DER input: length was not minimally-encoded" + ) + if length < 0x80: + # If the length could have been encoded in short form, it must + # not use long form. + raise ValueError( + "Invalid DER input: length was not minimally-encoded" + ) + body = self.read_bytes(length) + return tag, DERReader(body) + + def read_element(self, expected_tag): + tag, body = self.read_any_element() + if tag != expected_tag: + raise ValueError("Invalid DER input: unexpected tag") + return body + + def read_single_element(self, expected_tag): + with self: + return self.read_element(expected_tag) + + def read_optional_element(self, expected_tag): + if len(self.data) > 0 and six.indexbytes(self.data, 0) == expected_tag: + return self.read_element(expected_tag) + return None + + def as_integer(self): + if len(self.data) == 0: + raise ValueError("Invalid DER input: empty integer contents") + first = six.indexbytes(self.data, 0) + if first & 0x80 == 0x80: + raise ValueError("Negative DER integers are not supported") + # The first 9 bits must not all be zero or all be ones. Otherwise, the + # encoding should have been one byte shorter. + if len(self.data) > 1: + second = six.indexbytes(self.data, 1) + if first == 0 and second & 0x80 == 0: + raise ValueError( + "Invalid DER input: integer not minimally-encoded" + ) + return int_from_bytes(self.data, "big") + + +def encode_der_integer(x): + if not isinstance(x, six.integer_types): + raise ValueError("Value must be an integer") + if x < 0: + raise ValueError("Negative integers are not supported") + n = x.bit_length() // 8 + 1 + return int_to_bytes(x, n) + + +def encode_der(tag, *children): + length = 0 + for child in children: + length += len(child) + chunks = [six.int2byte(tag)] + if length < 0x80: + chunks.append(six.int2byte(length)) + else: + length_bytes = int_to_bytes(length) + chunks.append(six.int2byte(0x80 | len(length_bytes))) + chunks.append(length_bytes) + chunks.extend(children) + return b"".join(chunks) diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/_oid.py b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/_oid.py new file mode 100644 index 0000000..de2771a --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/_oid.py @@ -0,0 +1,77 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function + +from cryptography import utils + + +class ObjectIdentifier(object): + def __init__(self, dotted_string): + self._dotted_string = dotted_string + + nodes = self._dotted_string.split(".") + intnodes = [] + + # There must be at least 2 nodes, the first node must be 0..2, and + # if less than 2, the second node cannot have a value outside the + # range 0..39. All nodes must be integers. + for node in nodes: + try: + node_value = int(node, 10) + except ValueError: + raise ValueError( + "Malformed OID: %s (non-integer nodes)" + % (self._dotted_string) + ) + if node_value < 0: + raise ValueError( + "Malformed OID: %s (negative-integer nodes)" + % (self._dotted_string) + ) + intnodes.append(node_value) + + if len(nodes) < 2: + raise ValueError( + "Malformed OID: %s (insufficient number of nodes)" + % (self._dotted_string) + ) + + if intnodes[0] > 2: + raise ValueError( + "Malformed OID: %s (first node outside valid range)" + % (self._dotted_string) + ) + + if intnodes[0] < 2 and intnodes[1] >= 40: + raise ValueError( + "Malformed OID: %s (second node outside valid range)" + % (self._dotted_string) + ) + + def __eq__(self, other): + if not isinstance(other, ObjectIdentifier): + return NotImplemented + + return self.dotted_string == other.dotted_string + + def __ne__(self, other): + return not self == other + + def __repr__(self): + return "".format( + self.dotted_string, self._name + ) + + def __hash__(self): + return hash(self.dotted_string) + + @property + def _name(self): + # Lazy import to avoid an import cycle + from cryptography.x509.oid import _OID_NAMES + + return _OID_NAMES.get(self, "Unknown OID") + + dotted_string = utils.read_only_property("_dotted_string") diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/__init__.py b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/__init__.py new file mode 100644 index 0000000..1563936 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/__init__.py @@ -0,0 +1,26 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function + + +_default_backend = None + + +def default_backend(): + global _default_backend + + if _default_backend is None: + from cryptography.hazmat.backends.openssl.backend import backend + + _default_backend = backend + + return _default_backend + + +def _get_backend(backend): + if backend is None: + return default_backend() + else: + return backend diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/__pycache__/__init__.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/__pycache__/__init__.cpython-39.pyc new file mode 100644 index 0000000..4b19871 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/__pycache__/__init__.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/__pycache__/interfaces.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/__pycache__/interfaces.cpython-39.pyc new file mode 100644 index 0000000..1395bd8 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/__pycache__/interfaces.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/interfaces.py b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/interfaces.py new file mode 100644 index 0000000..418980a --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/interfaces.py @@ -0,0 +1,396 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function + +import abc + +import six + + +@six.add_metaclass(abc.ABCMeta) +class CipherBackend(object): + @abc.abstractmethod + def cipher_supported(self, cipher, mode): + """ + Return True if the given cipher and mode are supported. + """ + + @abc.abstractmethod + def create_symmetric_encryption_ctx(self, cipher, mode): + """ + Get a CipherContext that can be used for encryption. + """ + + @abc.abstractmethod + def create_symmetric_decryption_ctx(self, cipher, mode): + """ + Get a CipherContext that can be used for decryption. + """ + + +@six.add_metaclass(abc.ABCMeta) +class HashBackend(object): + @abc.abstractmethod + def hash_supported(self, algorithm): + """ + Return True if the hash algorithm is supported by this backend. + """ + + @abc.abstractmethod + def create_hash_ctx(self, algorithm): + """ + Create a HashContext for calculating a message digest. + """ + + +@six.add_metaclass(abc.ABCMeta) +class HMACBackend(object): + @abc.abstractmethod + def hmac_supported(self, algorithm): + """ + Return True if the hash algorithm is supported for HMAC by this + backend. + """ + + @abc.abstractmethod + def create_hmac_ctx(self, key, algorithm): + """ + Create a context for calculating a message authentication code. + """ + + +@six.add_metaclass(abc.ABCMeta) +class CMACBackend(object): + @abc.abstractmethod + def cmac_algorithm_supported(self, algorithm): + """ + Returns True if the block cipher is supported for CMAC by this backend + """ + + @abc.abstractmethod + def create_cmac_ctx(self, algorithm): + """ + Create a context for calculating a message authentication code. + """ + + +@six.add_metaclass(abc.ABCMeta) +class PBKDF2HMACBackend(object): + @abc.abstractmethod + def pbkdf2_hmac_supported(self, algorithm): + """ + Return True if the hash algorithm is supported for PBKDF2 by this + backend. + """ + + @abc.abstractmethod + def derive_pbkdf2_hmac( + self, algorithm, length, salt, iterations, key_material + ): + """ + Return length bytes derived from provided PBKDF2 parameters. + """ + + +@six.add_metaclass(abc.ABCMeta) +class RSABackend(object): + @abc.abstractmethod + def generate_rsa_private_key(self, public_exponent, key_size): + """ + Generate an RSAPrivateKey instance with public_exponent and a modulus + of key_size bits. + """ + + @abc.abstractmethod + def rsa_padding_supported(self, padding): + """ + Returns True if the backend supports the given padding options. + """ + + @abc.abstractmethod + def generate_rsa_parameters_supported(self, public_exponent, key_size): + """ + Returns True if the backend supports the given parameters for key + generation. + """ + + @abc.abstractmethod + def load_rsa_private_numbers(self, numbers): + """ + Returns an RSAPrivateKey provider. + """ + + @abc.abstractmethod + def load_rsa_public_numbers(self, numbers): + """ + Returns an RSAPublicKey provider. + """ + + +@six.add_metaclass(abc.ABCMeta) +class DSABackend(object): + @abc.abstractmethod + def generate_dsa_parameters(self, key_size): + """ + Generate a DSAParameters instance with a modulus of key_size bits. + """ + + @abc.abstractmethod + def generate_dsa_private_key(self, parameters): + """ + Generate a DSAPrivateKey instance with parameters as a DSAParameters + object. + """ + + @abc.abstractmethod + def generate_dsa_private_key_and_parameters(self, key_size): + """ + Generate a DSAPrivateKey instance using key size only. + """ + + @abc.abstractmethod + def dsa_hash_supported(self, algorithm): + """ + Return True if the hash algorithm is supported by the backend for DSA. + """ + + @abc.abstractmethod + def dsa_parameters_supported(self, p, q, g): + """ + Return True if the parameters are supported by the backend for DSA. + """ + + @abc.abstractmethod + def load_dsa_private_numbers(self, numbers): + """ + Returns a DSAPrivateKey provider. + """ + + @abc.abstractmethod + def load_dsa_public_numbers(self, numbers): + """ + Returns a DSAPublicKey provider. + """ + + @abc.abstractmethod + def load_dsa_parameter_numbers(self, numbers): + """ + Returns a DSAParameters provider. + """ + + +@six.add_metaclass(abc.ABCMeta) +class EllipticCurveBackend(object): + @abc.abstractmethod + def elliptic_curve_signature_algorithm_supported( + self, signature_algorithm, curve + ): + """ + Returns True if the backend supports the named elliptic curve with the + specified signature algorithm. + """ + + @abc.abstractmethod + def elliptic_curve_supported(self, curve): + """ + Returns True if the backend supports the named elliptic curve. + """ + + @abc.abstractmethod + def generate_elliptic_curve_private_key(self, curve): + """ + Return an object conforming to the EllipticCurvePrivateKey interface. + """ + + @abc.abstractmethod + def load_elliptic_curve_public_numbers(self, numbers): + """ + Return an EllipticCurvePublicKey provider using the given numbers. + """ + + @abc.abstractmethod + def load_elliptic_curve_private_numbers(self, numbers): + """ + Return an EllipticCurvePrivateKey provider using the given numbers. + """ + + @abc.abstractmethod + def elliptic_curve_exchange_algorithm_supported(self, algorithm, curve): + """ + Returns whether the exchange algorithm is supported by this backend. + """ + + @abc.abstractmethod + def derive_elliptic_curve_private_key(self, private_value, curve): + """ + Compute the private key given the private value and curve. + """ + + +@six.add_metaclass(abc.ABCMeta) +class PEMSerializationBackend(object): + @abc.abstractmethod + def load_pem_private_key(self, data, password): + """ + Loads a private key from PEM encoded data, using the provided password + if the data is encrypted. + """ + + @abc.abstractmethod + def load_pem_public_key(self, data): + """ + Loads a public key from PEM encoded data. + """ + + @abc.abstractmethod + def load_pem_parameters(self, data): + """ + Load encryption parameters from PEM encoded data. + """ + + +@six.add_metaclass(abc.ABCMeta) +class DERSerializationBackend(object): + @abc.abstractmethod + def load_der_private_key(self, data, password): + """ + Loads a private key from DER encoded data. Uses the provided password + if the data is encrypted. + """ + + @abc.abstractmethod + def load_der_public_key(self, data): + """ + Loads a public key from DER encoded data. + """ + + @abc.abstractmethod + def load_der_parameters(self, data): + """ + Load encryption parameters from DER encoded data. + """ + + +@six.add_metaclass(abc.ABCMeta) +class X509Backend(object): + @abc.abstractmethod + def load_pem_x509_certificate(self, data): + """ + Load an X.509 certificate from PEM encoded data. + """ + + @abc.abstractmethod + def load_der_x509_certificate(self, data): + """ + Load an X.509 certificate from DER encoded data. + """ + + @abc.abstractmethod + def load_der_x509_csr(self, data): + """ + Load an X.509 CSR from DER encoded data. + """ + + @abc.abstractmethod + def load_pem_x509_csr(self, data): + """ + Load an X.509 CSR from PEM encoded data. + """ + + @abc.abstractmethod + def create_x509_csr(self, builder, private_key, algorithm): + """ + Create and sign an X.509 CSR from a CSR builder object. + """ + + @abc.abstractmethod + def create_x509_certificate(self, builder, private_key, algorithm): + """ + Create and sign an X.509 certificate from a CertificateBuilder object. + """ + + @abc.abstractmethod + def create_x509_crl(self, builder, private_key, algorithm): + """ + Create and sign an X.509 CertificateRevocationList from a + CertificateRevocationListBuilder object. + """ + + @abc.abstractmethod + def create_x509_revoked_certificate(self, builder): + """ + Create a RevokedCertificate object from a RevokedCertificateBuilder + object. + """ + + @abc.abstractmethod + def x509_name_bytes(self, name): + """ + Compute the DER encoded bytes of an X509 Name object. + """ + + +@six.add_metaclass(abc.ABCMeta) +class DHBackend(object): + @abc.abstractmethod + def generate_dh_parameters(self, generator, key_size): + """ + Generate a DHParameters instance with a modulus of key_size bits. + Using the given generator. Often 2 or 5. + """ + + @abc.abstractmethod + def generate_dh_private_key(self, parameters): + """ + Generate a DHPrivateKey instance with parameters as a DHParameters + object. + """ + + @abc.abstractmethod + def generate_dh_private_key_and_parameters(self, generator, key_size): + """ + Generate a DHPrivateKey instance using key size only. + Using the given generator. Often 2 or 5. + """ + + @abc.abstractmethod + def load_dh_private_numbers(self, numbers): + """ + Load a DHPrivateKey from DHPrivateNumbers + """ + + @abc.abstractmethod + def load_dh_public_numbers(self, numbers): + """ + Load a DHPublicKey from DHPublicNumbers. + """ + + @abc.abstractmethod + def load_dh_parameter_numbers(self, numbers): + """ + Load DHParameters from DHParameterNumbers. + """ + + @abc.abstractmethod + def dh_parameters_supported(self, p, g, q=None): + """ + Returns whether the backend supports DH with these parameter values. + """ + + @abc.abstractmethod + def dh_x942_serialization_supported(self): + """ + Returns True if the backend supports the serialization of DH objects + with subgroup order (q). + """ + + +@six.add_metaclass(abc.ABCMeta) +class ScryptBackend(object): + @abc.abstractmethod + def derive_scrypt(self, key_material, salt, length, n, r, p): + """ + Return bytes derived from provided Scrypt parameters. + """ diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/openssl/__init__.py b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/openssl/__init__.py new file mode 100644 index 0000000..8eadeb6 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/openssl/__init__.py @@ -0,0 +1,10 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function + +from cryptography.hazmat.backends.openssl.backend import backend + + +__all__ = ["backend"] diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/openssl/__pycache__/__init__.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/openssl/__pycache__/__init__.cpython-39.pyc new file mode 100644 index 0000000..e66b486 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/openssl/__pycache__/__init__.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/openssl/__pycache__/aead.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/openssl/__pycache__/aead.cpython-39.pyc new file mode 100644 index 0000000..88a1366 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/openssl/__pycache__/aead.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/openssl/__pycache__/backend.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/openssl/__pycache__/backend.cpython-39.pyc new file mode 100644 index 0000000..16d55a0 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/openssl/__pycache__/backend.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/openssl/__pycache__/ciphers.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/openssl/__pycache__/ciphers.cpython-39.pyc new file mode 100644 index 0000000..752630f Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/openssl/__pycache__/ciphers.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/openssl/__pycache__/cmac.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/openssl/__pycache__/cmac.cpython-39.pyc new file mode 100644 index 0000000..492c041 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/openssl/__pycache__/cmac.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/openssl/__pycache__/decode_asn1.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/openssl/__pycache__/decode_asn1.cpython-39.pyc new file mode 100644 index 0000000..40a04b4 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/openssl/__pycache__/decode_asn1.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/openssl/__pycache__/dh.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/openssl/__pycache__/dh.cpython-39.pyc new file mode 100644 index 0000000..4d9d178 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/openssl/__pycache__/dh.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/openssl/__pycache__/dsa.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/openssl/__pycache__/dsa.cpython-39.pyc new file mode 100644 index 0000000..a42100d Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/openssl/__pycache__/dsa.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/openssl/__pycache__/ec.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/openssl/__pycache__/ec.cpython-39.pyc new file mode 100644 index 0000000..0ab63ec Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/openssl/__pycache__/ec.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/openssl/__pycache__/ed25519.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/openssl/__pycache__/ed25519.cpython-39.pyc new file mode 100644 index 0000000..d7c285e Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/openssl/__pycache__/ed25519.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/openssl/__pycache__/ed448.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/openssl/__pycache__/ed448.cpython-39.pyc new file mode 100644 index 0000000..9eb7939 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/openssl/__pycache__/ed448.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/openssl/__pycache__/encode_asn1.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/openssl/__pycache__/encode_asn1.cpython-39.pyc new file mode 100644 index 0000000..4dfc206 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/openssl/__pycache__/encode_asn1.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/openssl/__pycache__/hashes.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/openssl/__pycache__/hashes.cpython-39.pyc new file mode 100644 index 0000000..de75c98 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/openssl/__pycache__/hashes.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/openssl/__pycache__/hmac.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/openssl/__pycache__/hmac.cpython-39.pyc new file mode 100644 index 0000000..cd4d58c Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/openssl/__pycache__/hmac.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/openssl/__pycache__/ocsp.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/openssl/__pycache__/ocsp.cpython-39.pyc new file mode 100644 index 0000000..293c922 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/openssl/__pycache__/ocsp.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/openssl/__pycache__/poly1305.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/openssl/__pycache__/poly1305.cpython-39.pyc new file mode 100644 index 0000000..ff46a4e Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/openssl/__pycache__/poly1305.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/openssl/__pycache__/rsa.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/openssl/__pycache__/rsa.cpython-39.pyc new file mode 100644 index 0000000..cddeaaa Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/openssl/__pycache__/rsa.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/openssl/__pycache__/utils.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/openssl/__pycache__/utils.cpython-39.pyc new file mode 100644 index 0000000..cd9e0c0 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/openssl/__pycache__/utils.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/openssl/__pycache__/x25519.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/openssl/__pycache__/x25519.cpython-39.pyc new file mode 100644 index 0000000..4ab901b Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/openssl/__pycache__/x25519.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/openssl/__pycache__/x448.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/openssl/__pycache__/x448.cpython-39.pyc new file mode 100644 index 0000000..5756353 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/openssl/__pycache__/x448.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/openssl/__pycache__/x509.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/openssl/__pycache__/x509.cpython-39.pyc new file mode 100644 index 0000000..2b5ecc1 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/openssl/__pycache__/x509.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/openssl/aead.py b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/openssl/aead.py new file mode 100644 index 0000000..4494916 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/openssl/aead.py @@ -0,0 +1,166 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function + +from cryptography.exceptions import InvalidTag + + +_ENCRYPT = 1 +_DECRYPT = 0 + + +def _aead_cipher_name(cipher): + from cryptography.hazmat.primitives.ciphers.aead import ( + AESCCM, + AESGCM, + ChaCha20Poly1305, + ) + + if isinstance(cipher, ChaCha20Poly1305): + return b"chacha20-poly1305" + elif isinstance(cipher, AESCCM): + return "aes-{}-ccm".format(len(cipher._key) * 8).encode("ascii") + else: + assert isinstance(cipher, AESGCM) + return "aes-{}-gcm".format(len(cipher._key) * 8).encode("ascii") + + +def _aead_setup(backend, cipher_name, key, nonce, tag, tag_len, operation): + evp_cipher = backend._lib.EVP_get_cipherbyname(cipher_name) + backend.openssl_assert(evp_cipher != backend._ffi.NULL) + ctx = backend._lib.EVP_CIPHER_CTX_new() + ctx = backend._ffi.gc(ctx, backend._lib.EVP_CIPHER_CTX_free) + res = backend._lib.EVP_CipherInit_ex( + ctx, + evp_cipher, + backend._ffi.NULL, + backend._ffi.NULL, + backend._ffi.NULL, + int(operation == _ENCRYPT), + ) + backend.openssl_assert(res != 0) + res = backend._lib.EVP_CIPHER_CTX_set_key_length(ctx, len(key)) + backend.openssl_assert(res != 0) + res = backend._lib.EVP_CIPHER_CTX_ctrl( + ctx, + backend._lib.EVP_CTRL_AEAD_SET_IVLEN, + len(nonce), + backend._ffi.NULL, + ) + backend.openssl_assert(res != 0) + if operation == _DECRYPT: + res = backend._lib.EVP_CIPHER_CTX_ctrl( + ctx, backend._lib.EVP_CTRL_AEAD_SET_TAG, len(tag), tag + ) + backend.openssl_assert(res != 0) + elif cipher_name.endswith(b"-ccm"): + res = backend._lib.EVP_CIPHER_CTX_ctrl( + ctx, backend._lib.EVP_CTRL_AEAD_SET_TAG, tag_len, backend._ffi.NULL + ) + backend.openssl_assert(res != 0) + + nonce_ptr = backend._ffi.from_buffer(nonce) + key_ptr = backend._ffi.from_buffer(key) + res = backend._lib.EVP_CipherInit_ex( + ctx, + backend._ffi.NULL, + backend._ffi.NULL, + key_ptr, + nonce_ptr, + int(operation == _ENCRYPT), + ) + backend.openssl_assert(res != 0) + return ctx + + +def _set_length(backend, ctx, data_len): + intptr = backend._ffi.new("int *") + res = backend._lib.EVP_CipherUpdate( + ctx, backend._ffi.NULL, intptr, backend._ffi.NULL, data_len + ) + backend.openssl_assert(res != 0) + + +def _process_aad(backend, ctx, associated_data): + outlen = backend._ffi.new("int *") + res = backend._lib.EVP_CipherUpdate( + ctx, backend._ffi.NULL, outlen, associated_data, len(associated_data) + ) + backend.openssl_assert(res != 0) + + +def _process_data(backend, ctx, data): + outlen = backend._ffi.new("int *") + buf = backend._ffi.new("unsigned char[]", len(data)) + res = backend._lib.EVP_CipherUpdate(ctx, buf, outlen, data, len(data)) + backend.openssl_assert(res != 0) + return backend._ffi.buffer(buf, outlen[0])[:] + + +def _encrypt(backend, cipher, nonce, data, associated_data, tag_length): + from cryptography.hazmat.primitives.ciphers.aead import AESCCM + + cipher_name = _aead_cipher_name(cipher) + ctx = _aead_setup( + backend, cipher_name, cipher._key, nonce, None, tag_length, _ENCRYPT + ) + # CCM requires us to pass the length of the data before processing anything + # However calling this with any other AEAD results in an error + if isinstance(cipher, AESCCM): + _set_length(backend, ctx, len(data)) + + _process_aad(backend, ctx, associated_data) + processed_data = _process_data(backend, ctx, data) + outlen = backend._ffi.new("int *") + res = backend._lib.EVP_CipherFinal_ex(ctx, backend._ffi.NULL, outlen) + backend.openssl_assert(res != 0) + backend.openssl_assert(outlen[0] == 0) + tag_buf = backend._ffi.new("unsigned char[]", tag_length) + res = backend._lib.EVP_CIPHER_CTX_ctrl( + ctx, backend._lib.EVP_CTRL_AEAD_GET_TAG, tag_length, tag_buf + ) + backend.openssl_assert(res != 0) + tag = backend._ffi.buffer(tag_buf)[:] + + return processed_data + tag + + +def _decrypt(backend, cipher, nonce, data, associated_data, tag_length): + from cryptography.hazmat.primitives.ciphers.aead import AESCCM + + if len(data) < tag_length: + raise InvalidTag + tag = data[-tag_length:] + data = data[:-tag_length] + cipher_name = _aead_cipher_name(cipher) + ctx = _aead_setup( + backend, cipher_name, cipher._key, nonce, tag, tag_length, _DECRYPT + ) + # CCM requires us to pass the length of the data before processing anything + # However calling this with any other AEAD results in an error + if isinstance(cipher, AESCCM): + _set_length(backend, ctx, len(data)) + + _process_aad(backend, ctx, associated_data) + # CCM has a different error path if the tag doesn't match. Errors are + # raised in Update and Final is irrelevant. + if isinstance(cipher, AESCCM): + outlen = backend._ffi.new("int *") + buf = backend._ffi.new("unsigned char[]", len(data)) + res = backend._lib.EVP_CipherUpdate(ctx, buf, outlen, data, len(data)) + if res != 1: + backend._consume_errors() + raise InvalidTag + + processed_data = backend._ffi.buffer(buf, outlen[0])[:] + else: + processed_data = _process_data(backend, ctx, data) + outlen = backend._ffi.new("int *") + res = backend._lib.EVP_CipherFinal_ex(ctx, backend._ffi.NULL, outlen) + if res == 0: + backend._consume_errors() + raise InvalidTag + + return processed_data diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/openssl/backend.py b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/openssl/backend.py new file mode 100644 index 0000000..45d4a1a --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/openssl/backend.py @@ -0,0 +1,2776 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function + +import collections +import contextlib +import itertools +import warnings +from contextlib import contextmanager + +import six +from six.moves import range + +from cryptography import utils, x509 +from cryptography.exceptions import UnsupportedAlgorithm, _Reasons +from cryptography.hazmat._der import ( + INTEGER, + NULL, + SEQUENCE, + encode_der, + encode_der_integer, +) +from cryptography.hazmat.backends.interfaces import ( + CMACBackend, + CipherBackend, + DERSerializationBackend, + DHBackend, + DSABackend, + EllipticCurveBackend, + HMACBackend, + HashBackend, + PBKDF2HMACBackend, + PEMSerializationBackend, + RSABackend, + ScryptBackend, + X509Backend, +) +from cryptography.hazmat.backends.openssl import aead +from cryptography.hazmat.backends.openssl.ciphers import _CipherContext +from cryptography.hazmat.backends.openssl.cmac import _CMACContext +from cryptography.hazmat.backends.openssl.decode_asn1 import ( + _CRL_ENTRY_REASON_ENUM_TO_CODE, + _CRL_EXTENSION_HANDLERS, + _EXTENSION_HANDLERS_BASE, + _EXTENSION_HANDLERS_SCT, + _OCSP_BASICRESP_EXTENSION_HANDLERS, + _OCSP_REQ_EXTENSION_HANDLERS, + _OCSP_SINGLERESP_EXTENSION_HANDLERS_SCT, + _REVOKED_EXTENSION_HANDLERS, + _X509ExtensionParser, +) +from cryptography.hazmat.backends.openssl.dh import ( + _DHParameters, + _DHPrivateKey, + _DHPublicKey, + _dh_params_dup, +) +from cryptography.hazmat.backends.openssl.dsa import ( + _DSAParameters, + _DSAPrivateKey, + _DSAPublicKey, +) +from cryptography.hazmat.backends.openssl.ec import ( + _EllipticCurvePrivateKey, + _EllipticCurvePublicKey, +) +from cryptography.hazmat.backends.openssl.ed25519 import ( + _Ed25519PrivateKey, + _Ed25519PublicKey, +) +from cryptography.hazmat.backends.openssl.ed448 import ( + _ED448_KEY_SIZE, + _Ed448PrivateKey, + _Ed448PublicKey, +) +from cryptography.hazmat.backends.openssl.encode_asn1 import ( + _CRL_ENTRY_EXTENSION_ENCODE_HANDLERS, + _CRL_EXTENSION_ENCODE_HANDLERS, + _EXTENSION_ENCODE_HANDLERS, + _OCSP_BASICRESP_EXTENSION_ENCODE_HANDLERS, + _OCSP_REQUEST_EXTENSION_ENCODE_HANDLERS, + _encode_asn1_int_gc, + _encode_asn1_str_gc, + _encode_name_gc, + _txt2obj_gc, +) +from cryptography.hazmat.backends.openssl.hashes import _HashContext +from cryptography.hazmat.backends.openssl.hmac import _HMACContext +from cryptography.hazmat.backends.openssl.ocsp import ( + _OCSPRequest, + _OCSPResponse, +) +from cryptography.hazmat.backends.openssl.poly1305 import ( + _POLY1305_KEY_SIZE, + _Poly1305Context, +) +from cryptography.hazmat.backends.openssl.rsa import ( + _RSAPrivateKey, + _RSAPublicKey, +) +from cryptography.hazmat.backends.openssl.x25519 import ( + _X25519PrivateKey, + _X25519PublicKey, +) +from cryptography.hazmat.backends.openssl.x448 import ( + _X448PrivateKey, + _X448PublicKey, +) +from cryptography.hazmat.backends.openssl.x509 import ( + _Certificate, + _CertificateRevocationList, + _CertificateSigningRequest, + _RevokedCertificate, +) +from cryptography.hazmat.bindings.openssl import binding +from cryptography.hazmat.primitives import hashes, serialization +from cryptography.hazmat.primitives.asymmetric import ( + dh, + dsa, + ec, + ed25519, + ed448, + rsa, +) +from cryptography.hazmat.primitives.asymmetric.padding import ( + MGF1, + OAEP, + PKCS1v15, + PSS, +) +from cryptography.hazmat.primitives.ciphers.algorithms import ( + AES, + ARC4, + Blowfish, + CAST5, + Camellia, + ChaCha20, + IDEA, + SEED, + TripleDES, +) +from cryptography.hazmat.primitives.ciphers.modes import ( + CBC, + CFB, + CFB8, + CTR, + ECB, + GCM, + OFB, + XTS, +) +from cryptography.hazmat.primitives.kdf import scrypt +from cryptography.hazmat.primitives.serialization import pkcs7, ssh +from cryptography.x509 import ocsp + + +_MemoryBIO = collections.namedtuple("_MemoryBIO", ["bio", "char_ptr"]) + + +# Not actually supported, just used as a marker for some serialization tests. +class _RC2(object): + pass + + +@utils.register_interface(CipherBackend) +@utils.register_interface(CMACBackend) +@utils.register_interface(DERSerializationBackend) +@utils.register_interface(DHBackend) +@utils.register_interface(DSABackend) +@utils.register_interface(EllipticCurveBackend) +@utils.register_interface(HashBackend) +@utils.register_interface(HMACBackend) +@utils.register_interface(PBKDF2HMACBackend) +@utils.register_interface(RSABackend) +@utils.register_interface(PEMSerializationBackend) +@utils.register_interface(X509Backend) +@utils.register_interface_if( + binding.Binding().lib.Cryptography_HAS_SCRYPT, ScryptBackend +) +class Backend(object): + """ + OpenSSL API binding interfaces. + """ + + name = "openssl" + + # FIPS has opinions about acceptable algorithms and key sizes, but the + # disallowed algorithms are still present in OpenSSL. They just error if + # you try to use them. To avoid that we allowlist the algorithms in + # FIPS 140-3. This isn't ideal, but FIPS 140-3 is trash so here we are. + _fips_aead = { + b"aes-128-ccm", + b"aes-192-ccm", + b"aes-256-ccm", + b"aes-128-gcm", + b"aes-192-gcm", + b"aes-256-gcm", + } + _fips_ciphers = (AES, TripleDES) + _fips_hashes = ( + hashes.SHA1, + hashes.SHA224, + hashes.SHA256, + hashes.SHA384, + hashes.SHA512, + hashes.SHA512_224, + hashes.SHA512_256, + hashes.SHA3_224, + hashes.SHA3_256, + hashes.SHA3_384, + hashes.SHA3_512, + hashes.SHAKE128, + hashes.SHAKE256, + ) + _fips_rsa_min_key_size = 2048 + _fips_rsa_min_public_exponent = 65537 + _fips_dsa_min_modulus = 1 << 2048 + _fips_dh_min_key_size = 2048 + _fips_dh_min_modulus = 1 << _fips_dh_min_key_size + + def __init__(self): + self._binding = binding.Binding() + self._ffi = self._binding.ffi + self._lib = self._binding.lib + self._fips_enabled = self._is_fips_enabled() + + self._cipher_registry = {} + self._register_default_ciphers() + self._register_x509_ext_parsers() + self._register_x509_encoders() + if self._fips_enabled and self._lib.CRYPTOGRAPHY_NEEDS_OSRANDOM_ENGINE: + warnings.warn( + "OpenSSL FIPS mode is enabled. Can't enable DRBG fork safety.", + UserWarning, + ) + else: + self.activate_osrandom_engine() + self._dh_types = [self._lib.EVP_PKEY_DH] + if self._lib.Cryptography_HAS_EVP_PKEY_DHX: + self._dh_types.append(self._lib.EVP_PKEY_DHX) + + def openssl_assert(self, ok, errors=None): + return binding._openssl_assert(self._lib, ok, errors=errors) + + def _is_fips_enabled(self): + fips_mode = getattr(self._lib, "FIPS_mode", lambda: 0) + mode = fips_mode() + if mode == 0: + # OpenSSL without FIPS pushes an error on the error stack + self._lib.ERR_clear_error() + return bool(mode) + + def activate_builtin_random(self): + if self._lib.CRYPTOGRAPHY_NEEDS_OSRANDOM_ENGINE: + # Obtain a new structural reference. + e = self._lib.ENGINE_get_default_RAND() + if e != self._ffi.NULL: + self._lib.ENGINE_unregister_RAND(e) + # Reset the RNG to use the built-in. + res = self._lib.RAND_set_rand_method(self._ffi.NULL) + self.openssl_assert(res == 1) + # decrement the structural reference from get_default_RAND + res = self._lib.ENGINE_finish(e) + self.openssl_assert(res == 1) + + @contextlib.contextmanager + def _get_osurandom_engine(self): + # Fetches an engine by id and returns it. This creates a structural + # reference. + e = self._lib.ENGINE_by_id(self._lib.Cryptography_osrandom_engine_id) + self.openssl_assert(e != self._ffi.NULL) + # Initialize the engine for use. This adds a functional reference. + res = self._lib.ENGINE_init(e) + self.openssl_assert(res == 1) + + try: + yield e + finally: + # Decrement the structural ref incremented by ENGINE_by_id. + res = self._lib.ENGINE_free(e) + self.openssl_assert(res == 1) + # Decrement the functional ref incremented by ENGINE_init. + res = self._lib.ENGINE_finish(e) + self.openssl_assert(res == 1) + + def activate_osrandom_engine(self): + if self._lib.CRYPTOGRAPHY_NEEDS_OSRANDOM_ENGINE: + # Unregister and free the current engine. + self.activate_builtin_random() + with self._get_osurandom_engine() as e: + # Set the engine as the default RAND provider. + res = self._lib.ENGINE_set_default_RAND(e) + self.openssl_assert(res == 1) + # Reset the RNG to use the engine + res = self._lib.RAND_set_rand_method(self._ffi.NULL) + self.openssl_assert(res == 1) + + def osrandom_engine_implementation(self): + buf = self._ffi.new("char[]", 64) + with self._get_osurandom_engine() as e: + res = self._lib.ENGINE_ctrl_cmd( + e, b"get_implementation", len(buf), buf, self._ffi.NULL, 0 + ) + self.openssl_assert(res > 0) + return self._ffi.string(buf).decode("ascii") + + def openssl_version_text(self): + """ + Friendly string name of the loaded OpenSSL library. This is not + necessarily the same version as it was compiled against. + + Example: OpenSSL 1.1.1d 10 Sep 2019 + """ + return self._ffi.string( + self._lib.OpenSSL_version(self._lib.OPENSSL_VERSION) + ).decode("ascii") + + def openssl_version_number(self): + return self._lib.OpenSSL_version_num() + + def create_hmac_ctx(self, key, algorithm): + return _HMACContext(self, key, algorithm) + + def _evp_md_from_algorithm(self, algorithm): + if algorithm.name == "blake2b" or algorithm.name == "blake2s": + alg = "{}{}".format( + algorithm.name, algorithm.digest_size * 8 + ).encode("ascii") + else: + alg = algorithm.name.encode("ascii") + + evp_md = self._lib.EVP_get_digestbyname(alg) + return evp_md + + def _evp_md_non_null_from_algorithm(self, algorithm): + evp_md = self._evp_md_from_algorithm(algorithm) + self.openssl_assert(evp_md != self._ffi.NULL) + return evp_md + + def hash_supported(self, algorithm): + if self._fips_enabled and not isinstance(algorithm, self._fips_hashes): + return False + + evp_md = self._evp_md_from_algorithm(algorithm) + return evp_md != self._ffi.NULL + + def hmac_supported(self, algorithm): + return self.hash_supported(algorithm) + + def create_hash_ctx(self, algorithm): + return _HashContext(self, algorithm) + + def cipher_supported(self, cipher, mode): + if self._fips_enabled and not isinstance(cipher, self._fips_ciphers): + return False + try: + adapter = self._cipher_registry[type(cipher), type(mode)] + except KeyError: + return False + evp_cipher = adapter(self, cipher, mode) + return self._ffi.NULL != evp_cipher + + def register_cipher_adapter(self, cipher_cls, mode_cls, adapter): + if (cipher_cls, mode_cls) in self._cipher_registry: + raise ValueError( + "Duplicate registration for: {} {}.".format( + cipher_cls, mode_cls + ) + ) + self._cipher_registry[cipher_cls, mode_cls] = adapter + + def _register_default_ciphers(self): + for mode_cls in [CBC, CTR, ECB, OFB, CFB, CFB8, GCM]: + self.register_cipher_adapter( + AES, + mode_cls, + GetCipherByName("{cipher.name}-{cipher.key_size}-{mode.name}"), + ) + for mode_cls in [CBC, CTR, ECB, OFB, CFB]: + self.register_cipher_adapter( + Camellia, + mode_cls, + GetCipherByName("{cipher.name}-{cipher.key_size}-{mode.name}"), + ) + for mode_cls in [CBC, CFB, CFB8, OFB]: + self.register_cipher_adapter( + TripleDES, mode_cls, GetCipherByName("des-ede3-{mode.name}") + ) + self.register_cipher_adapter( + TripleDES, ECB, GetCipherByName("des-ede3") + ) + for mode_cls in [CBC, CFB, OFB, ECB]: + self.register_cipher_adapter( + Blowfish, mode_cls, GetCipherByName("bf-{mode.name}") + ) + for mode_cls in [CBC, CFB, OFB, ECB]: + self.register_cipher_adapter( + SEED, mode_cls, GetCipherByName("seed-{mode.name}") + ) + for cipher_cls, mode_cls in itertools.product( + [CAST5, IDEA], + [CBC, OFB, CFB, ECB], + ): + self.register_cipher_adapter( + cipher_cls, + mode_cls, + GetCipherByName("{cipher.name}-{mode.name}"), + ) + self.register_cipher_adapter(ARC4, type(None), GetCipherByName("rc4")) + # We don't actually support RC2, this is just used by some tests. + self.register_cipher_adapter(_RC2, type(None), GetCipherByName("rc2")) + self.register_cipher_adapter( + ChaCha20, type(None), GetCipherByName("chacha20") + ) + self.register_cipher_adapter(AES, XTS, _get_xts_cipher) + + def _register_x509_ext_parsers(self): + ext_handlers = _EXTENSION_HANDLERS_BASE.copy() + # All revoked extensions are valid single response extensions, see: + # https://tools.ietf.org/html/rfc6960#section-4.4.5 + singleresp_handlers = _REVOKED_EXTENSION_HANDLERS.copy() + + if self._lib.Cryptography_HAS_SCT: + ext_handlers.update(_EXTENSION_HANDLERS_SCT) + singleresp_handlers.update(_OCSP_SINGLERESP_EXTENSION_HANDLERS_SCT) + + self._certificate_extension_parser = _X509ExtensionParser( + self, + ext_count=self._lib.X509_get_ext_count, + get_ext=self._lib.X509_get_ext, + handlers=ext_handlers, + ) + self._csr_extension_parser = _X509ExtensionParser( + self, + ext_count=self._lib.sk_X509_EXTENSION_num, + get_ext=self._lib.sk_X509_EXTENSION_value, + handlers=ext_handlers, + ) + self._revoked_cert_extension_parser = _X509ExtensionParser( + self, + ext_count=self._lib.X509_REVOKED_get_ext_count, + get_ext=self._lib.X509_REVOKED_get_ext, + handlers=_REVOKED_EXTENSION_HANDLERS, + ) + self._crl_extension_parser = _X509ExtensionParser( + self, + ext_count=self._lib.X509_CRL_get_ext_count, + get_ext=self._lib.X509_CRL_get_ext, + handlers=_CRL_EXTENSION_HANDLERS, + ) + self._ocsp_req_ext_parser = _X509ExtensionParser( + self, + ext_count=self._lib.OCSP_REQUEST_get_ext_count, + get_ext=self._lib.OCSP_REQUEST_get_ext, + handlers=_OCSP_REQ_EXTENSION_HANDLERS, + ) + self._ocsp_basicresp_ext_parser = _X509ExtensionParser( + self, + ext_count=self._lib.OCSP_BASICRESP_get_ext_count, + get_ext=self._lib.OCSP_BASICRESP_get_ext, + handlers=_OCSP_BASICRESP_EXTENSION_HANDLERS, + ) + self._ocsp_singleresp_ext_parser = _X509ExtensionParser( + self, + ext_count=self._lib.OCSP_SINGLERESP_get_ext_count, + get_ext=self._lib.OCSP_SINGLERESP_get_ext, + handlers=singleresp_handlers, + ) + + def _register_x509_encoders(self): + self._extension_encode_handlers = _EXTENSION_ENCODE_HANDLERS.copy() + self._crl_extension_encode_handlers = ( + _CRL_EXTENSION_ENCODE_HANDLERS.copy() + ) + self._crl_entry_extension_encode_handlers = ( + _CRL_ENTRY_EXTENSION_ENCODE_HANDLERS.copy() + ) + self._ocsp_request_extension_encode_handlers = ( + _OCSP_REQUEST_EXTENSION_ENCODE_HANDLERS.copy() + ) + self._ocsp_basicresp_extension_encode_handlers = ( + _OCSP_BASICRESP_EXTENSION_ENCODE_HANDLERS.copy() + ) + + def create_symmetric_encryption_ctx(self, cipher, mode): + return _CipherContext(self, cipher, mode, _CipherContext._ENCRYPT) + + def create_symmetric_decryption_ctx(self, cipher, mode): + return _CipherContext(self, cipher, mode, _CipherContext._DECRYPT) + + def pbkdf2_hmac_supported(self, algorithm): + return self.hmac_supported(algorithm) + + def derive_pbkdf2_hmac( + self, algorithm, length, salt, iterations, key_material + ): + buf = self._ffi.new("unsigned char[]", length) + evp_md = self._evp_md_non_null_from_algorithm(algorithm) + key_material_ptr = self._ffi.from_buffer(key_material) + res = self._lib.PKCS5_PBKDF2_HMAC( + key_material_ptr, + len(key_material), + salt, + len(salt), + iterations, + evp_md, + length, + buf, + ) + self.openssl_assert(res == 1) + return self._ffi.buffer(buf)[:] + + def _consume_errors(self): + return binding._consume_errors(self._lib) + + def _consume_errors_with_text(self): + return binding._consume_errors_with_text(self._lib) + + def _bn_to_int(self, bn): + assert bn != self._ffi.NULL + + if not six.PY2: + # Python 3 has constant time from_bytes, so use that. + bn_num_bytes = self._lib.BN_num_bytes(bn) + bin_ptr = self._ffi.new("unsigned char[]", bn_num_bytes) + bin_len = self._lib.BN_bn2bin(bn, bin_ptr) + # A zero length means the BN has value 0 + self.openssl_assert(bin_len >= 0) + val = int.from_bytes(self._ffi.buffer(bin_ptr)[:bin_len], "big") + if self._lib.BN_is_negative(bn): + val = -val + return val + else: + # Under Python 2 the best we can do is hex() + hex_cdata = self._lib.BN_bn2hex(bn) + self.openssl_assert(hex_cdata != self._ffi.NULL) + hex_str = self._ffi.string(hex_cdata) + self._lib.OPENSSL_free(hex_cdata) + return int(hex_str, 16) + + def _int_to_bn(self, num, bn=None): + """ + Converts a python integer to a BIGNUM. The returned BIGNUM will not + be garbage collected (to support adding them to structs that take + ownership of the object). Be sure to register it for GC if it will + be discarded after use. + """ + assert bn is None or bn != self._ffi.NULL + + if bn is None: + bn = self._ffi.NULL + + if not six.PY2: + # Python 3 has constant time to_bytes, so use that. + + binary = num.to_bytes(int(num.bit_length() / 8.0 + 1), "big") + bn_ptr = self._lib.BN_bin2bn(binary, len(binary), bn) + self.openssl_assert(bn_ptr != self._ffi.NULL) + return bn_ptr + + else: + # Under Python 2 the best we can do is hex(), [2:] removes the 0x + # prefix. + hex_num = hex(num).rstrip("L")[2:].encode("ascii") + bn_ptr = self._ffi.new("BIGNUM **") + bn_ptr[0] = bn + res = self._lib.BN_hex2bn(bn_ptr, hex_num) + self.openssl_assert(res != 0) + self.openssl_assert(bn_ptr[0] != self._ffi.NULL) + return bn_ptr[0] + + def generate_rsa_private_key(self, public_exponent, key_size): + rsa._verify_rsa_parameters(public_exponent, key_size) + + rsa_cdata = self._lib.RSA_new() + self.openssl_assert(rsa_cdata != self._ffi.NULL) + rsa_cdata = self._ffi.gc(rsa_cdata, self._lib.RSA_free) + + bn = self._int_to_bn(public_exponent) + bn = self._ffi.gc(bn, self._lib.BN_free) + + res = self._lib.RSA_generate_key_ex( + rsa_cdata, key_size, bn, self._ffi.NULL + ) + self.openssl_assert(res == 1) + evp_pkey = self._rsa_cdata_to_evp_pkey(rsa_cdata) + + return _RSAPrivateKey(self, rsa_cdata, evp_pkey) + + def generate_rsa_parameters_supported(self, public_exponent, key_size): + return ( + public_exponent >= 3 + and public_exponent & 1 != 0 + and key_size >= 512 + ) + + def load_rsa_private_numbers(self, numbers): + rsa._check_private_key_components( + numbers.p, + numbers.q, + numbers.d, + numbers.dmp1, + numbers.dmq1, + numbers.iqmp, + numbers.public_numbers.e, + numbers.public_numbers.n, + ) + rsa_cdata = self._lib.RSA_new() + self.openssl_assert(rsa_cdata != self._ffi.NULL) + rsa_cdata = self._ffi.gc(rsa_cdata, self._lib.RSA_free) + p = self._int_to_bn(numbers.p) + q = self._int_to_bn(numbers.q) + d = self._int_to_bn(numbers.d) + dmp1 = self._int_to_bn(numbers.dmp1) + dmq1 = self._int_to_bn(numbers.dmq1) + iqmp = self._int_to_bn(numbers.iqmp) + e = self._int_to_bn(numbers.public_numbers.e) + n = self._int_to_bn(numbers.public_numbers.n) + res = self._lib.RSA_set0_factors(rsa_cdata, p, q) + self.openssl_assert(res == 1) + res = self._lib.RSA_set0_key(rsa_cdata, n, e, d) + self.openssl_assert(res == 1) + res = self._lib.RSA_set0_crt_params(rsa_cdata, dmp1, dmq1, iqmp) + self.openssl_assert(res == 1) + evp_pkey = self._rsa_cdata_to_evp_pkey(rsa_cdata) + + return _RSAPrivateKey(self, rsa_cdata, evp_pkey) + + def load_rsa_public_numbers(self, numbers): + rsa._check_public_key_components(numbers.e, numbers.n) + rsa_cdata = self._lib.RSA_new() + self.openssl_assert(rsa_cdata != self._ffi.NULL) + rsa_cdata = self._ffi.gc(rsa_cdata, self._lib.RSA_free) + e = self._int_to_bn(numbers.e) + n = self._int_to_bn(numbers.n) + res = self._lib.RSA_set0_key(rsa_cdata, n, e, self._ffi.NULL) + self.openssl_assert(res == 1) + evp_pkey = self._rsa_cdata_to_evp_pkey(rsa_cdata) + + return _RSAPublicKey(self, rsa_cdata, evp_pkey) + + def _create_evp_pkey_gc(self): + evp_pkey = self._lib.EVP_PKEY_new() + self.openssl_assert(evp_pkey != self._ffi.NULL) + evp_pkey = self._ffi.gc(evp_pkey, self._lib.EVP_PKEY_free) + return evp_pkey + + def _rsa_cdata_to_evp_pkey(self, rsa_cdata): + evp_pkey = self._create_evp_pkey_gc() + res = self._lib.EVP_PKEY_set1_RSA(evp_pkey, rsa_cdata) + self.openssl_assert(res == 1) + return evp_pkey + + def _bytes_to_bio(self, data): + """ + Return a _MemoryBIO namedtuple of (BIO, char*). + + The char* is the storage for the BIO and it must stay alive until the + BIO is finished with. + """ + data_ptr = self._ffi.from_buffer(data) + bio = self._lib.BIO_new_mem_buf(data_ptr, len(data)) + self.openssl_assert(bio != self._ffi.NULL) + + return _MemoryBIO(self._ffi.gc(bio, self._lib.BIO_free), data_ptr) + + def _create_mem_bio_gc(self): + """ + Creates an empty memory BIO. + """ + bio_method = self._lib.BIO_s_mem() + self.openssl_assert(bio_method != self._ffi.NULL) + bio = self._lib.BIO_new(bio_method) + self.openssl_assert(bio != self._ffi.NULL) + bio = self._ffi.gc(bio, self._lib.BIO_free) + return bio + + def _read_mem_bio(self, bio): + """ + Reads a memory BIO. This only works on memory BIOs. + """ + buf = self._ffi.new("char **") + buf_len = self._lib.BIO_get_mem_data(bio, buf) + self.openssl_assert(buf_len > 0) + self.openssl_assert(buf[0] != self._ffi.NULL) + bio_data = self._ffi.buffer(buf[0], buf_len)[:] + return bio_data + + def _evp_pkey_to_private_key(self, evp_pkey): + """ + Return the appropriate type of PrivateKey given an evp_pkey cdata + pointer. + """ + + key_type = self._lib.EVP_PKEY_id(evp_pkey) + + if key_type == self._lib.EVP_PKEY_RSA: + rsa_cdata = self._lib.EVP_PKEY_get1_RSA(evp_pkey) + self.openssl_assert(rsa_cdata != self._ffi.NULL) + rsa_cdata = self._ffi.gc(rsa_cdata, self._lib.RSA_free) + return _RSAPrivateKey(self, rsa_cdata, evp_pkey) + elif key_type == self._lib.EVP_PKEY_DSA: + dsa_cdata = self._lib.EVP_PKEY_get1_DSA(evp_pkey) + self.openssl_assert(dsa_cdata != self._ffi.NULL) + dsa_cdata = self._ffi.gc(dsa_cdata, self._lib.DSA_free) + return _DSAPrivateKey(self, dsa_cdata, evp_pkey) + elif key_type == self._lib.EVP_PKEY_EC: + ec_cdata = self._lib.EVP_PKEY_get1_EC_KEY(evp_pkey) + self.openssl_assert(ec_cdata != self._ffi.NULL) + ec_cdata = self._ffi.gc(ec_cdata, self._lib.EC_KEY_free) + return _EllipticCurvePrivateKey(self, ec_cdata, evp_pkey) + elif key_type in self._dh_types: + dh_cdata = self._lib.EVP_PKEY_get1_DH(evp_pkey) + self.openssl_assert(dh_cdata != self._ffi.NULL) + dh_cdata = self._ffi.gc(dh_cdata, self._lib.DH_free) + return _DHPrivateKey(self, dh_cdata, evp_pkey) + elif key_type == getattr(self._lib, "EVP_PKEY_ED25519", None): + # EVP_PKEY_ED25519 is not present in OpenSSL < 1.1.1 + return _Ed25519PrivateKey(self, evp_pkey) + elif key_type == getattr(self._lib, "EVP_PKEY_X448", None): + # EVP_PKEY_X448 is not present in OpenSSL < 1.1.1 + return _X448PrivateKey(self, evp_pkey) + elif key_type == getattr(self._lib, "EVP_PKEY_X25519", None): + # EVP_PKEY_X25519 is not present in OpenSSL < 1.1.0 + return _X25519PrivateKey(self, evp_pkey) + elif key_type == getattr(self._lib, "EVP_PKEY_ED448", None): + # EVP_PKEY_ED448 is not present in OpenSSL < 1.1.1 + return _Ed448PrivateKey(self, evp_pkey) + else: + raise UnsupportedAlgorithm("Unsupported key type.") + + def _evp_pkey_to_public_key(self, evp_pkey): + """ + Return the appropriate type of PublicKey given an evp_pkey cdata + pointer. + """ + + key_type = self._lib.EVP_PKEY_id(evp_pkey) + + if key_type == self._lib.EVP_PKEY_RSA: + rsa_cdata = self._lib.EVP_PKEY_get1_RSA(evp_pkey) + self.openssl_assert(rsa_cdata != self._ffi.NULL) + rsa_cdata = self._ffi.gc(rsa_cdata, self._lib.RSA_free) + return _RSAPublicKey(self, rsa_cdata, evp_pkey) + elif key_type == self._lib.EVP_PKEY_DSA: + dsa_cdata = self._lib.EVP_PKEY_get1_DSA(evp_pkey) + self.openssl_assert(dsa_cdata != self._ffi.NULL) + dsa_cdata = self._ffi.gc(dsa_cdata, self._lib.DSA_free) + return _DSAPublicKey(self, dsa_cdata, evp_pkey) + elif key_type == self._lib.EVP_PKEY_EC: + ec_cdata = self._lib.EVP_PKEY_get1_EC_KEY(evp_pkey) + self.openssl_assert(ec_cdata != self._ffi.NULL) + ec_cdata = self._ffi.gc(ec_cdata, self._lib.EC_KEY_free) + return _EllipticCurvePublicKey(self, ec_cdata, evp_pkey) + elif key_type in self._dh_types: + dh_cdata = self._lib.EVP_PKEY_get1_DH(evp_pkey) + self.openssl_assert(dh_cdata != self._ffi.NULL) + dh_cdata = self._ffi.gc(dh_cdata, self._lib.DH_free) + return _DHPublicKey(self, dh_cdata, evp_pkey) + elif key_type == getattr(self._lib, "EVP_PKEY_ED25519", None): + # EVP_PKEY_ED25519 is not present in OpenSSL < 1.1.1 + return _Ed25519PublicKey(self, evp_pkey) + elif key_type == getattr(self._lib, "EVP_PKEY_X448", None): + # EVP_PKEY_X448 is not present in OpenSSL < 1.1.1 + return _X448PublicKey(self, evp_pkey) + elif key_type == getattr(self._lib, "EVP_PKEY_X25519", None): + # EVP_PKEY_X25519 is not present in OpenSSL < 1.1.0 + return _X25519PublicKey(self, evp_pkey) + elif key_type == getattr(self._lib, "EVP_PKEY_ED448", None): + # EVP_PKEY_X25519 is not present in OpenSSL < 1.1.1 + return _Ed448PublicKey(self, evp_pkey) + else: + raise UnsupportedAlgorithm("Unsupported key type.") + + def _oaep_hash_supported(self, algorithm): + if self._lib.Cryptography_HAS_RSA_OAEP_MD: + return isinstance( + algorithm, + ( + hashes.SHA1, + hashes.SHA224, + hashes.SHA256, + hashes.SHA384, + hashes.SHA512, + ), + ) + else: + return isinstance(algorithm, hashes.SHA1) + + def rsa_padding_supported(self, padding): + if isinstance(padding, PKCS1v15): + return True + elif isinstance(padding, PSS) and isinstance(padding._mgf, MGF1): + return self.hash_supported(padding._mgf._algorithm) + elif isinstance(padding, OAEP) and isinstance(padding._mgf, MGF1): + return ( + self._oaep_hash_supported(padding._mgf._algorithm) + and self._oaep_hash_supported(padding._algorithm) + and ( + (padding._label is None or len(padding._label) == 0) + or self._lib.Cryptography_HAS_RSA_OAEP_LABEL == 1 + ) + ) + else: + return False + + def generate_dsa_parameters(self, key_size): + if key_size not in (1024, 2048, 3072, 4096): + raise ValueError( + "Key size must be 1024, 2048, 3072, or 4096 bits." + ) + + ctx = self._lib.DSA_new() + self.openssl_assert(ctx != self._ffi.NULL) + ctx = self._ffi.gc(ctx, self._lib.DSA_free) + + res = self._lib.DSA_generate_parameters_ex( + ctx, + key_size, + self._ffi.NULL, + 0, + self._ffi.NULL, + self._ffi.NULL, + self._ffi.NULL, + ) + + self.openssl_assert(res == 1) + + return _DSAParameters(self, ctx) + + def generate_dsa_private_key(self, parameters): + ctx = self._lib.DSAparams_dup(parameters._dsa_cdata) + self.openssl_assert(ctx != self._ffi.NULL) + ctx = self._ffi.gc(ctx, self._lib.DSA_free) + self._lib.DSA_generate_key(ctx) + evp_pkey = self._dsa_cdata_to_evp_pkey(ctx) + + return _DSAPrivateKey(self, ctx, evp_pkey) + + def generate_dsa_private_key_and_parameters(self, key_size): + parameters = self.generate_dsa_parameters(key_size) + return self.generate_dsa_private_key(parameters) + + def _dsa_cdata_set_values(self, dsa_cdata, p, q, g, pub_key, priv_key): + res = self._lib.DSA_set0_pqg(dsa_cdata, p, q, g) + self.openssl_assert(res == 1) + res = self._lib.DSA_set0_key(dsa_cdata, pub_key, priv_key) + self.openssl_assert(res == 1) + + def load_dsa_private_numbers(self, numbers): + dsa._check_dsa_private_numbers(numbers) + parameter_numbers = numbers.public_numbers.parameter_numbers + + dsa_cdata = self._lib.DSA_new() + self.openssl_assert(dsa_cdata != self._ffi.NULL) + dsa_cdata = self._ffi.gc(dsa_cdata, self._lib.DSA_free) + + p = self._int_to_bn(parameter_numbers.p) + q = self._int_to_bn(parameter_numbers.q) + g = self._int_to_bn(parameter_numbers.g) + pub_key = self._int_to_bn(numbers.public_numbers.y) + priv_key = self._int_to_bn(numbers.x) + self._dsa_cdata_set_values(dsa_cdata, p, q, g, pub_key, priv_key) + + evp_pkey = self._dsa_cdata_to_evp_pkey(dsa_cdata) + + return _DSAPrivateKey(self, dsa_cdata, evp_pkey) + + def load_dsa_public_numbers(self, numbers): + dsa._check_dsa_parameters(numbers.parameter_numbers) + dsa_cdata = self._lib.DSA_new() + self.openssl_assert(dsa_cdata != self._ffi.NULL) + dsa_cdata = self._ffi.gc(dsa_cdata, self._lib.DSA_free) + + p = self._int_to_bn(numbers.parameter_numbers.p) + q = self._int_to_bn(numbers.parameter_numbers.q) + g = self._int_to_bn(numbers.parameter_numbers.g) + pub_key = self._int_to_bn(numbers.y) + priv_key = self._ffi.NULL + self._dsa_cdata_set_values(dsa_cdata, p, q, g, pub_key, priv_key) + + evp_pkey = self._dsa_cdata_to_evp_pkey(dsa_cdata) + + return _DSAPublicKey(self, dsa_cdata, evp_pkey) + + def load_dsa_parameter_numbers(self, numbers): + dsa._check_dsa_parameters(numbers) + dsa_cdata = self._lib.DSA_new() + self.openssl_assert(dsa_cdata != self._ffi.NULL) + dsa_cdata = self._ffi.gc(dsa_cdata, self._lib.DSA_free) + + p = self._int_to_bn(numbers.p) + q = self._int_to_bn(numbers.q) + g = self._int_to_bn(numbers.g) + res = self._lib.DSA_set0_pqg(dsa_cdata, p, q, g) + self.openssl_assert(res == 1) + + return _DSAParameters(self, dsa_cdata) + + def _dsa_cdata_to_evp_pkey(self, dsa_cdata): + evp_pkey = self._create_evp_pkey_gc() + res = self._lib.EVP_PKEY_set1_DSA(evp_pkey, dsa_cdata) + self.openssl_assert(res == 1) + return evp_pkey + + def dsa_hash_supported(self, algorithm): + return self.hash_supported(algorithm) + + def dsa_parameters_supported(self, p, q, g): + return True + + def cmac_algorithm_supported(self, algorithm): + return self.cipher_supported( + algorithm, CBC(b"\x00" * algorithm.block_size) + ) + + def create_cmac_ctx(self, algorithm): + return _CMACContext(self, algorithm) + + def _x509_check_signature_params(self, private_key, algorithm): + if isinstance( + private_key, (ed25519.Ed25519PrivateKey, ed448.Ed448PrivateKey) + ): + if algorithm is not None: + raise ValueError( + "algorithm must be None when signing via ed25519 or ed448" + ) + elif not isinstance( + private_key, + (rsa.RSAPrivateKey, dsa.DSAPrivateKey, ec.EllipticCurvePrivateKey), + ): + raise TypeError( + "Key must be an rsa, dsa, ec, ed25519, or ed448 private key." + ) + elif not isinstance(algorithm, hashes.HashAlgorithm): + raise TypeError("Algorithm must be a registered hash algorithm.") + elif isinstance(algorithm, hashes.MD5) and not isinstance( + private_key, rsa.RSAPrivateKey + ): + raise ValueError( + "MD5 hash algorithm is only supported with RSA keys" + ) + + def create_x509_csr(self, builder, private_key, algorithm): + if not isinstance(builder, x509.CertificateSigningRequestBuilder): + raise TypeError("Builder type mismatch.") + self._x509_check_signature_params(private_key, algorithm) + + # Resolve the signature algorithm. + evp_md = self._evp_md_x509_null_if_eddsa(private_key, algorithm) + + # Create an empty request. + x509_req = self._lib.X509_REQ_new() + self.openssl_assert(x509_req != self._ffi.NULL) + x509_req = self._ffi.gc(x509_req, self._lib.X509_REQ_free) + + # Set x509 version. + res = self._lib.X509_REQ_set_version(x509_req, x509.Version.v1.value) + self.openssl_assert(res == 1) + + # Set subject name. + res = self._lib.X509_REQ_set_subject_name( + x509_req, _encode_name_gc(self, builder._subject_name) + ) + self.openssl_assert(res == 1) + + # Set subject public key. + public_key = private_key.public_key() + res = self._lib.X509_REQ_set_pubkey(x509_req, public_key._evp_pkey) + self.openssl_assert(res == 1) + + # Add extensions. + sk_extension = self._lib.sk_X509_EXTENSION_new_null() + self.openssl_assert(sk_extension != self._ffi.NULL) + sk_extension = self._ffi.gc( + sk_extension, + lambda x: self._lib.sk_X509_EXTENSION_pop_free( + x, + self._ffi.addressof( + self._lib._original_lib, "X509_EXTENSION_free" + ), + ), + ) + # Don't GC individual extensions because the memory is owned by + # sk_extensions and will be freed along with it. + self._create_x509_extensions( + extensions=builder._extensions, + handlers=self._extension_encode_handlers, + x509_obj=sk_extension, + add_func=self._lib.sk_X509_EXTENSION_insert, + gc=False, + ) + res = self._lib.X509_REQ_add_extensions(x509_req, sk_extension) + self.openssl_assert(res == 1) + + # Add attributes (all bytes encoded as ASN1 UTF8_STRING) + for attr_oid, attr_val in builder._attributes: + obj = _txt2obj_gc(self, attr_oid.dotted_string) + res = self._lib.X509_REQ_add1_attr_by_OBJ( + x509_req, + obj, + x509.name._ASN1Type.UTF8String.value, + attr_val, + len(attr_val), + ) + self.openssl_assert(res == 1) + + # Sign the request using the requester's private key. + res = self._lib.X509_REQ_sign(x509_req, private_key._evp_pkey, evp_md) + if res == 0: + errors = self._consume_errors_with_text() + raise ValueError("Signing failed", errors) + + return _CertificateSigningRequest(self, x509_req) + + def create_x509_certificate(self, builder, private_key, algorithm): + if not isinstance(builder, x509.CertificateBuilder): + raise TypeError("Builder type mismatch.") + self._x509_check_signature_params(private_key, algorithm) + + # Resolve the signature algorithm. + evp_md = self._evp_md_x509_null_if_eddsa(private_key, algorithm) + + # Create an empty certificate. + x509_cert = self._lib.X509_new() + x509_cert = self._ffi.gc(x509_cert, self._lib.X509_free) + + # Set the x509 version. + res = self._lib.X509_set_version(x509_cert, builder._version.value) + self.openssl_assert(res == 1) + + # Set the subject's name. + res = self._lib.X509_set_subject_name( + x509_cert, _encode_name_gc(self, builder._subject_name) + ) + self.openssl_assert(res == 1) + + # Set the subject's public key. + res = self._lib.X509_set_pubkey( + x509_cert, builder._public_key._evp_pkey + ) + self.openssl_assert(res == 1) + + # Set the certificate serial number. + serial_number = _encode_asn1_int_gc(self, builder._serial_number) + res = self._lib.X509_set_serialNumber(x509_cert, serial_number) + self.openssl_assert(res == 1) + + # Set the "not before" time. + self._set_asn1_time( + self._lib.X509_getm_notBefore(x509_cert), builder._not_valid_before + ) + + # Set the "not after" time. + self._set_asn1_time( + self._lib.X509_getm_notAfter(x509_cert), builder._not_valid_after + ) + + # Add extensions. + self._create_x509_extensions( + extensions=builder._extensions, + handlers=self._extension_encode_handlers, + x509_obj=x509_cert, + add_func=self._lib.X509_add_ext, + gc=True, + ) + + # Set the issuer name. + res = self._lib.X509_set_issuer_name( + x509_cert, _encode_name_gc(self, builder._issuer_name) + ) + self.openssl_assert(res == 1) + + # Sign the certificate with the issuer's private key. + res = self._lib.X509_sign(x509_cert, private_key._evp_pkey, evp_md) + if res == 0: + errors = self._consume_errors_with_text() + raise ValueError("Signing failed", errors) + + return _Certificate(self, x509_cert) + + def _evp_md_x509_null_if_eddsa(self, private_key, algorithm): + if isinstance( + private_key, (ed25519.Ed25519PrivateKey, ed448.Ed448PrivateKey) + ): + # OpenSSL requires us to pass NULL for EVP_MD for ed25519/ed448 + return self._ffi.NULL + else: + return self._evp_md_non_null_from_algorithm(algorithm) + + def _set_asn1_time(self, asn1_time, time): + if time.year >= 2050: + asn1_str = time.strftime("%Y%m%d%H%M%SZ").encode("ascii") + else: + asn1_str = time.strftime("%y%m%d%H%M%SZ").encode("ascii") + res = self._lib.ASN1_TIME_set_string(asn1_time, asn1_str) + self.openssl_assert(res == 1) + + def _create_asn1_time(self, time): + asn1_time = self._lib.ASN1_TIME_new() + self.openssl_assert(asn1_time != self._ffi.NULL) + asn1_time = self._ffi.gc(asn1_time, self._lib.ASN1_TIME_free) + self._set_asn1_time(asn1_time, time) + return asn1_time + + def create_x509_crl(self, builder, private_key, algorithm): + if not isinstance(builder, x509.CertificateRevocationListBuilder): + raise TypeError("Builder type mismatch.") + self._x509_check_signature_params(private_key, algorithm) + + evp_md = self._evp_md_x509_null_if_eddsa(private_key, algorithm) + + # Create an empty CRL. + x509_crl = self._lib.X509_CRL_new() + x509_crl = self._ffi.gc(x509_crl, self._lib.X509_CRL_free) + + # Set the x509 CRL version. We only support v2 (integer value 1). + res = self._lib.X509_CRL_set_version(x509_crl, 1) + self.openssl_assert(res == 1) + + # Set the issuer name. + res = self._lib.X509_CRL_set_issuer_name( + x509_crl, _encode_name_gc(self, builder._issuer_name) + ) + self.openssl_assert(res == 1) + + # Set the last update time. + last_update = self._create_asn1_time(builder._last_update) + res = self._lib.X509_CRL_set_lastUpdate(x509_crl, last_update) + self.openssl_assert(res == 1) + + # Set the next update time. + next_update = self._create_asn1_time(builder._next_update) + res = self._lib.X509_CRL_set_nextUpdate(x509_crl, next_update) + self.openssl_assert(res == 1) + + # Add extensions. + self._create_x509_extensions( + extensions=builder._extensions, + handlers=self._crl_extension_encode_handlers, + x509_obj=x509_crl, + add_func=self._lib.X509_CRL_add_ext, + gc=True, + ) + + # add revoked certificates + for revoked_cert in builder._revoked_certificates: + # Duplicating because the X509_CRL takes ownership and will free + # this memory when X509_CRL_free is called. + revoked = self._lib.X509_REVOKED_dup(revoked_cert._x509_revoked) + self.openssl_assert(revoked != self._ffi.NULL) + res = self._lib.X509_CRL_add0_revoked(x509_crl, revoked) + self.openssl_assert(res == 1) + + res = self._lib.X509_CRL_sign(x509_crl, private_key._evp_pkey, evp_md) + if res == 0: + errors = self._consume_errors_with_text() + raise ValueError("Signing failed", errors) + + return _CertificateRevocationList(self, x509_crl) + + def _create_x509_extensions( + self, extensions, handlers, x509_obj, add_func, gc + ): + for i, extension in enumerate(extensions): + x509_extension = self._create_x509_extension(handlers, extension) + self.openssl_assert(x509_extension != self._ffi.NULL) + + if gc: + x509_extension = self._ffi.gc( + x509_extension, self._lib.X509_EXTENSION_free + ) + res = add_func(x509_obj, x509_extension, i) + self.openssl_assert(res >= 1) + + def _create_raw_x509_extension(self, extension, value): + obj = _txt2obj_gc(self, extension.oid.dotted_string) + return self._lib.X509_EXTENSION_create_by_OBJ( + self._ffi.NULL, obj, 1 if extension.critical else 0, value + ) + + def _create_x509_extension(self, handlers, extension): + if isinstance(extension.value, x509.UnrecognizedExtension): + value = _encode_asn1_str_gc(self, extension.value.value) + return self._create_raw_x509_extension(extension, value) + elif isinstance(extension.value, x509.TLSFeature): + asn1 = encode_der( + SEQUENCE, + *[ + encode_der(INTEGER, encode_der_integer(x.value)) + for x in extension.value + ] + ) + value = _encode_asn1_str_gc(self, asn1) + return self._create_raw_x509_extension(extension, value) + elif isinstance(extension.value, x509.PrecertPoison): + value = _encode_asn1_str_gc(self, encode_der(NULL)) + return self._create_raw_x509_extension(extension, value) + else: + try: + encode = handlers[extension.oid] + except KeyError: + raise NotImplementedError( + "Extension not supported: {}".format(extension.oid) + ) + + ext_struct = encode(self, extension.value) + nid = self._lib.OBJ_txt2nid( + extension.oid.dotted_string.encode("ascii") + ) + self.openssl_assert(nid != self._lib.NID_undef) + return self._lib.X509V3_EXT_i2d( + nid, 1 if extension.critical else 0, ext_struct + ) + + def create_x509_revoked_certificate(self, builder): + if not isinstance(builder, x509.RevokedCertificateBuilder): + raise TypeError("Builder type mismatch.") + + x509_revoked = self._lib.X509_REVOKED_new() + self.openssl_assert(x509_revoked != self._ffi.NULL) + x509_revoked = self._ffi.gc(x509_revoked, self._lib.X509_REVOKED_free) + serial_number = _encode_asn1_int_gc(self, builder._serial_number) + res = self._lib.X509_REVOKED_set_serialNumber( + x509_revoked, serial_number + ) + self.openssl_assert(res == 1) + rev_date = self._create_asn1_time(builder._revocation_date) + res = self._lib.X509_REVOKED_set_revocationDate(x509_revoked, rev_date) + self.openssl_assert(res == 1) + # add CRL entry extensions + self._create_x509_extensions( + extensions=builder._extensions, + handlers=self._crl_entry_extension_encode_handlers, + x509_obj=x509_revoked, + add_func=self._lib.X509_REVOKED_add_ext, + gc=True, + ) + return _RevokedCertificate(self, None, x509_revoked) + + def load_pem_private_key(self, data, password): + return self._load_key( + self._lib.PEM_read_bio_PrivateKey, + self._evp_pkey_to_private_key, + data, + password, + ) + + def load_pem_public_key(self, data): + mem_bio = self._bytes_to_bio(data) + evp_pkey = self._lib.PEM_read_bio_PUBKEY( + mem_bio.bio, self._ffi.NULL, self._ffi.NULL, self._ffi.NULL + ) + if evp_pkey != self._ffi.NULL: + evp_pkey = self._ffi.gc(evp_pkey, self._lib.EVP_PKEY_free) + return self._evp_pkey_to_public_key(evp_pkey) + else: + # It's not a (RSA/DSA/ECDSA) subjectPublicKeyInfo, but we still + # need to check to see if it is a pure PKCS1 RSA public key (not + # embedded in a subjectPublicKeyInfo) + self._consume_errors() + res = self._lib.BIO_reset(mem_bio.bio) + self.openssl_assert(res == 1) + rsa_cdata = self._lib.PEM_read_bio_RSAPublicKey( + mem_bio.bio, self._ffi.NULL, self._ffi.NULL, self._ffi.NULL + ) + if rsa_cdata != self._ffi.NULL: + rsa_cdata = self._ffi.gc(rsa_cdata, self._lib.RSA_free) + evp_pkey = self._rsa_cdata_to_evp_pkey(rsa_cdata) + return _RSAPublicKey(self, rsa_cdata, evp_pkey) + else: + self._handle_key_loading_error() + + def load_pem_parameters(self, data): + mem_bio = self._bytes_to_bio(data) + # only DH is supported currently + dh_cdata = self._lib.PEM_read_bio_DHparams( + mem_bio.bio, self._ffi.NULL, self._ffi.NULL, self._ffi.NULL + ) + if dh_cdata != self._ffi.NULL: + dh_cdata = self._ffi.gc(dh_cdata, self._lib.DH_free) + return _DHParameters(self, dh_cdata) + else: + self._handle_key_loading_error() + + def load_der_private_key(self, data, password): + # OpenSSL has a function called d2i_AutoPrivateKey that in theory + # handles this automatically, however it doesn't handle encrypted + # private keys. Instead we try to load the key two different ways. + # First we'll try to load it as a traditional key. + bio_data = self._bytes_to_bio(data) + key = self._evp_pkey_from_der_traditional_key(bio_data, password) + if key: + return self._evp_pkey_to_private_key(key) + else: + # Finally we try to load it with the method that handles encrypted + # PKCS8 properly. + return self._load_key( + self._lib.d2i_PKCS8PrivateKey_bio, + self._evp_pkey_to_private_key, + data, + password, + ) + + def _evp_pkey_from_der_traditional_key(self, bio_data, password): + key = self._lib.d2i_PrivateKey_bio(bio_data.bio, self._ffi.NULL) + if key != self._ffi.NULL: + key = self._ffi.gc(key, self._lib.EVP_PKEY_free) + if password is not None: + raise TypeError( + "Password was given but private key is not encrypted." + ) + + return key + else: + self._consume_errors() + return None + + def load_der_public_key(self, data): + mem_bio = self._bytes_to_bio(data) + evp_pkey = self._lib.d2i_PUBKEY_bio(mem_bio.bio, self._ffi.NULL) + if evp_pkey != self._ffi.NULL: + evp_pkey = self._ffi.gc(evp_pkey, self._lib.EVP_PKEY_free) + return self._evp_pkey_to_public_key(evp_pkey) + else: + # It's not a (RSA/DSA/ECDSA) subjectPublicKeyInfo, but we still + # need to check to see if it is a pure PKCS1 RSA public key (not + # embedded in a subjectPublicKeyInfo) + self._consume_errors() + res = self._lib.BIO_reset(mem_bio.bio) + self.openssl_assert(res == 1) + rsa_cdata = self._lib.d2i_RSAPublicKey_bio( + mem_bio.bio, self._ffi.NULL + ) + if rsa_cdata != self._ffi.NULL: + rsa_cdata = self._ffi.gc(rsa_cdata, self._lib.RSA_free) + evp_pkey = self._rsa_cdata_to_evp_pkey(rsa_cdata) + return _RSAPublicKey(self, rsa_cdata, evp_pkey) + else: + self._handle_key_loading_error() + + def load_der_parameters(self, data): + mem_bio = self._bytes_to_bio(data) + dh_cdata = self._lib.d2i_DHparams_bio(mem_bio.bio, self._ffi.NULL) + if dh_cdata != self._ffi.NULL: + dh_cdata = self._ffi.gc(dh_cdata, self._lib.DH_free) + return _DHParameters(self, dh_cdata) + elif self._lib.Cryptography_HAS_EVP_PKEY_DHX: + # We check to see if the is dhx. + self._consume_errors() + res = self._lib.BIO_reset(mem_bio.bio) + self.openssl_assert(res == 1) + dh_cdata = self._lib.Cryptography_d2i_DHxparams_bio( + mem_bio.bio, self._ffi.NULL + ) + if dh_cdata != self._ffi.NULL: + dh_cdata = self._ffi.gc(dh_cdata, self._lib.DH_free) + return _DHParameters(self, dh_cdata) + + self._handle_key_loading_error() + + def load_pem_x509_certificate(self, data): + mem_bio = self._bytes_to_bio(data) + x509 = self._lib.PEM_read_bio_X509( + mem_bio.bio, self._ffi.NULL, self._ffi.NULL, self._ffi.NULL + ) + if x509 == self._ffi.NULL: + self._consume_errors() + raise ValueError( + "Unable to load certificate. See https://cryptography.io/en/" + "latest/faq.html#why-can-t-i-import-my-pem-file for more" + " details." + ) + + x509 = self._ffi.gc(x509, self._lib.X509_free) + return _Certificate(self, x509) + + def load_der_x509_certificate(self, data): + mem_bio = self._bytes_to_bio(data) + x509 = self._lib.d2i_X509_bio(mem_bio.bio, self._ffi.NULL) + if x509 == self._ffi.NULL: + self._consume_errors() + raise ValueError("Unable to load certificate") + + x509 = self._ffi.gc(x509, self._lib.X509_free) + return _Certificate(self, x509) + + def load_pem_x509_crl(self, data): + mem_bio = self._bytes_to_bio(data) + x509_crl = self._lib.PEM_read_bio_X509_CRL( + mem_bio.bio, self._ffi.NULL, self._ffi.NULL, self._ffi.NULL + ) + if x509_crl == self._ffi.NULL: + self._consume_errors() + raise ValueError( + "Unable to load CRL. See https://cryptography.io/en/la" + "test/faq.html#why-can-t-i-import-my-pem-file for more" + " details." + ) + + x509_crl = self._ffi.gc(x509_crl, self._lib.X509_CRL_free) + return _CertificateRevocationList(self, x509_crl) + + def load_der_x509_crl(self, data): + mem_bio = self._bytes_to_bio(data) + x509_crl = self._lib.d2i_X509_CRL_bio(mem_bio.bio, self._ffi.NULL) + if x509_crl == self._ffi.NULL: + self._consume_errors() + raise ValueError("Unable to load CRL") + + x509_crl = self._ffi.gc(x509_crl, self._lib.X509_CRL_free) + return _CertificateRevocationList(self, x509_crl) + + def load_pem_x509_csr(self, data): + mem_bio = self._bytes_to_bio(data) + x509_req = self._lib.PEM_read_bio_X509_REQ( + mem_bio.bio, self._ffi.NULL, self._ffi.NULL, self._ffi.NULL + ) + if x509_req == self._ffi.NULL: + self._consume_errors() + raise ValueError( + "Unable to load request. See https://cryptography.io/en/" + "latest/faq.html#why-can-t-i-import-my-pem-file for more" + " details." + ) + + x509_req = self._ffi.gc(x509_req, self._lib.X509_REQ_free) + return _CertificateSigningRequest(self, x509_req) + + def load_der_x509_csr(self, data): + mem_bio = self._bytes_to_bio(data) + x509_req = self._lib.d2i_X509_REQ_bio(mem_bio.bio, self._ffi.NULL) + if x509_req == self._ffi.NULL: + self._consume_errors() + raise ValueError("Unable to load request") + + x509_req = self._ffi.gc(x509_req, self._lib.X509_REQ_free) + return _CertificateSigningRequest(self, x509_req) + + def _load_key(self, openssl_read_func, convert_func, data, password): + mem_bio = self._bytes_to_bio(data) + + userdata = self._ffi.new("CRYPTOGRAPHY_PASSWORD_DATA *") + if password is not None: + utils._check_byteslike("password", password) + password_ptr = self._ffi.from_buffer(password) + userdata.password = password_ptr + userdata.length = len(password) + + evp_pkey = openssl_read_func( + mem_bio.bio, + self._ffi.NULL, + self._ffi.addressof( + self._lib._original_lib, "Cryptography_pem_password_cb" + ), + userdata, + ) + + if evp_pkey == self._ffi.NULL: + if userdata.error != 0: + self._consume_errors() + if userdata.error == -1: + raise TypeError( + "Password was not given but private key is encrypted" + ) + else: + assert userdata.error == -2 + raise ValueError( + "Passwords longer than {} bytes are not supported " + "by this backend.".format(userdata.maxsize - 1) + ) + else: + self._handle_key_loading_error() + + evp_pkey = self._ffi.gc(evp_pkey, self._lib.EVP_PKEY_free) + + if password is not None and userdata.called == 0: + raise TypeError( + "Password was given but private key is not encrypted." + ) + + assert ( + password is not None and userdata.called == 1 + ) or password is None + + return convert_func(evp_pkey) + + def _handle_key_loading_error(self): + errors = self._consume_errors() + + if not errors: + raise ValueError( + "Could not deserialize key data. The data may be in an " + "incorrect format or it may be encrypted with an unsupported " + "algorithm." + ) + elif errors[0]._lib_reason_match( + self._lib.ERR_LIB_EVP, self._lib.EVP_R_BAD_DECRYPT + ) or errors[0]._lib_reason_match( + self._lib.ERR_LIB_PKCS12, + self._lib.PKCS12_R_PKCS12_CIPHERFINAL_ERROR, + ): + raise ValueError("Bad decrypt. Incorrect password?") + + elif any( + error._lib_reason_match( + self._lib.ERR_LIB_EVP, + self._lib.EVP_R_UNSUPPORTED_PRIVATE_KEY_ALGORITHM, + ) + for error in errors + ): + raise ValueError("Unsupported public key algorithm.") + + else: + raise ValueError( + "Could not deserialize key data. The data may be in an " + "incorrect format or it may be encrypted with an unsupported " + "algorithm." + ) + + def elliptic_curve_supported(self, curve): + try: + curve_nid = self._elliptic_curve_to_nid(curve) + except UnsupportedAlgorithm: + curve_nid = self._lib.NID_undef + + group = self._lib.EC_GROUP_new_by_curve_name(curve_nid) + + if group == self._ffi.NULL: + self._consume_errors() + return False + else: + self.openssl_assert(curve_nid != self._lib.NID_undef) + self._lib.EC_GROUP_free(group) + return True + + def elliptic_curve_signature_algorithm_supported( + self, signature_algorithm, curve + ): + # We only support ECDSA right now. + if not isinstance(signature_algorithm, ec.ECDSA): + return False + + return self.elliptic_curve_supported(curve) + + def generate_elliptic_curve_private_key(self, curve): + """ + Generate a new private key on the named curve. + """ + + if self.elliptic_curve_supported(curve): + ec_cdata = self._ec_key_new_by_curve(curve) + + res = self._lib.EC_KEY_generate_key(ec_cdata) + self.openssl_assert(res == 1) + + evp_pkey = self._ec_cdata_to_evp_pkey(ec_cdata) + + return _EllipticCurvePrivateKey(self, ec_cdata, evp_pkey) + else: + raise UnsupportedAlgorithm( + "Backend object does not support {}.".format(curve.name), + _Reasons.UNSUPPORTED_ELLIPTIC_CURVE, + ) + + def load_elliptic_curve_private_numbers(self, numbers): + public = numbers.public_numbers + + ec_cdata = self._ec_key_new_by_curve(public.curve) + + private_value = self._ffi.gc( + self._int_to_bn(numbers.private_value), self._lib.BN_clear_free + ) + res = self._lib.EC_KEY_set_private_key(ec_cdata, private_value) + self.openssl_assert(res == 1) + + ec_cdata = self._ec_key_set_public_key_affine_coordinates( + ec_cdata, public.x, public.y + ) + + evp_pkey = self._ec_cdata_to_evp_pkey(ec_cdata) + + return _EllipticCurvePrivateKey(self, ec_cdata, evp_pkey) + + def load_elliptic_curve_public_numbers(self, numbers): + ec_cdata = self._ec_key_new_by_curve(numbers.curve) + ec_cdata = self._ec_key_set_public_key_affine_coordinates( + ec_cdata, numbers.x, numbers.y + ) + evp_pkey = self._ec_cdata_to_evp_pkey(ec_cdata) + + return _EllipticCurvePublicKey(self, ec_cdata, evp_pkey) + + def load_elliptic_curve_public_bytes(self, curve, point_bytes): + ec_cdata = self._ec_key_new_by_curve(curve) + group = self._lib.EC_KEY_get0_group(ec_cdata) + self.openssl_assert(group != self._ffi.NULL) + point = self._lib.EC_POINT_new(group) + self.openssl_assert(point != self._ffi.NULL) + point = self._ffi.gc(point, self._lib.EC_POINT_free) + with self._tmp_bn_ctx() as bn_ctx: + res = self._lib.EC_POINT_oct2point( + group, point, point_bytes, len(point_bytes), bn_ctx + ) + if res != 1: + self._consume_errors() + raise ValueError("Invalid public bytes for the given curve") + + res = self._lib.EC_KEY_set_public_key(ec_cdata, point) + self.openssl_assert(res == 1) + evp_pkey = self._ec_cdata_to_evp_pkey(ec_cdata) + return _EllipticCurvePublicKey(self, ec_cdata, evp_pkey) + + def derive_elliptic_curve_private_key(self, private_value, curve): + ec_cdata = self._ec_key_new_by_curve(curve) + + get_func, group = self._ec_key_determine_group_get_func(ec_cdata) + + point = self._lib.EC_POINT_new(group) + self.openssl_assert(point != self._ffi.NULL) + point = self._ffi.gc(point, self._lib.EC_POINT_free) + + value = self._int_to_bn(private_value) + value = self._ffi.gc(value, self._lib.BN_clear_free) + + with self._tmp_bn_ctx() as bn_ctx: + res = self._lib.EC_POINT_mul( + group, point, value, self._ffi.NULL, self._ffi.NULL, bn_ctx + ) + self.openssl_assert(res == 1) + + bn_x = self._lib.BN_CTX_get(bn_ctx) + bn_y = self._lib.BN_CTX_get(bn_ctx) + + res = get_func(group, point, bn_x, bn_y, bn_ctx) + self.openssl_assert(res == 1) + + res = self._lib.EC_KEY_set_public_key(ec_cdata, point) + self.openssl_assert(res == 1) + private = self._int_to_bn(private_value) + private = self._ffi.gc(private, self._lib.BN_clear_free) + res = self._lib.EC_KEY_set_private_key(ec_cdata, private) + self.openssl_assert(res == 1) + + evp_pkey = self._ec_cdata_to_evp_pkey(ec_cdata) + + return _EllipticCurvePrivateKey(self, ec_cdata, evp_pkey) + + def _ec_key_new_by_curve(self, curve): + curve_nid = self._elliptic_curve_to_nid(curve) + return self._ec_key_new_by_curve_nid(curve_nid) + + def _ec_key_new_by_curve_nid(self, curve_nid): + ec_cdata = self._lib.EC_KEY_new_by_curve_name(curve_nid) + self.openssl_assert(ec_cdata != self._ffi.NULL) + return self._ffi.gc(ec_cdata, self._lib.EC_KEY_free) + + def load_der_ocsp_request(self, data): + mem_bio = self._bytes_to_bio(data) + request = self._lib.d2i_OCSP_REQUEST_bio(mem_bio.bio, self._ffi.NULL) + if request == self._ffi.NULL: + self._consume_errors() + raise ValueError("Unable to load OCSP request") + + request = self._ffi.gc(request, self._lib.OCSP_REQUEST_free) + return _OCSPRequest(self, request) + + def load_der_ocsp_response(self, data): + mem_bio = self._bytes_to_bio(data) + response = self._lib.d2i_OCSP_RESPONSE_bio(mem_bio.bio, self._ffi.NULL) + if response == self._ffi.NULL: + self._consume_errors() + raise ValueError("Unable to load OCSP response") + + response = self._ffi.gc(response, self._lib.OCSP_RESPONSE_free) + return _OCSPResponse(self, response) + + def create_ocsp_request(self, builder): + ocsp_req = self._lib.OCSP_REQUEST_new() + self.openssl_assert(ocsp_req != self._ffi.NULL) + ocsp_req = self._ffi.gc(ocsp_req, self._lib.OCSP_REQUEST_free) + cert, issuer, algorithm = builder._request + evp_md = self._evp_md_non_null_from_algorithm(algorithm) + certid = self._lib.OCSP_cert_to_id(evp_md, cert._x509, issuer._x509) + self.openssl_assert(certid != self._ffi.NULL) + onereq = self._lib.OCSP_request_add0_id(ocsp_req, certid) + self.openssl_assert(onereq != self._ffi.NULL) + self._create_x509_extensions( + extensions=builder._extensions, + handlers=self._ocsp_request_extension_encode_handlers, + x509_obj=ocsp_req, + add_func=self._lib.OCSP_REQUEST_add_ext, + gc=True, + ) + return _OCSPRequest(self, ocsp_req) + + def _create_ocsp_basic_response(self, builder, private_key, algorithm): + self._x509_check_signature_params(private_key, algorithm) + + basic = self._lib.OCSP_BASICRESP_new() + self.openssl_assert(basic != self._ffi.NULL) + basic = self._ffi.gc(basic, self._lib.OCSP_BASICRESP_free) + evp_md = self._evp_md_non_null_from_algorithm( + builder._response._algorithm + ) + certid = self._lib.OCSP_cert_to_id( + evp_md, + builder._response._cert._x509, + builder._response._issuer._x509, + ) + self.openssl_assert(certid != self._ffi.NULL) + certid = self._ffi.gc(certid, self._lib.OCSP_CERTID_free) + if builder._response._revocation_reason is None: + reason = -1 + else: + reason = _CRL_ENTRY_REASON_ENUM_TO_CODE[ + builder._response._revocation_reason + ] + if builder._response._revocation_time is None: + rev_time = self._ffi.NULL + else: + rev_time = self._create_asn1_time( + builder._response._revocation_time + ) + + next_update = self._ffi.NULL + if builder._response._next_update is not None: + next_update = self._create_asn1_time( + builder._response._next_update + ) + + this_update = self._create_asn1_time(builder._response._this_update) + + res = self._lib.OCSP_basic_add1_status( + basic, + certid, + builder._response._cert_status.value, + reason, + rev_time, + this_update, + next_update, + ) + self.openssl_assert(res != self._ffi.NULL) + # okay, now sign the basic structure + evp_md = self._evp_md_x509_null_if_eddsa(private_key, algorithm) + responder_cert, responder_encoding = builder._responder_id + flags = self._lib.OCSP_NOCERTS + if responder_encoding is ocsp.OCSPResponderEncoding.HASH: + flags |= self._lib.OCSP_RESPID_KEY + + if builder._certs is not None: + for cert in builder._certs: + res = self._lib.OCSP_basic_add1_cert(basic, cert._x509) + self.openssl_assert(res == 1) + + self._create_x509_extensions( + extensions=builder._extensions, + handlers=self._ocsp_basicresp_extension_encode_handlers, + x509_obj=basic, + add_func=self._lib.OCSP_BASICRESP_add_ext, + gc=True, + ) + + res = self._lib.OCSP_basic_sign( + basic, + responder_cert._x509, + private_key._evp_pkey, + evp_md, + self._ffi.NULL, + flags, + ) + if res != 1: + errors = self._consume_errors_with_text() + raise ValueError( + "Error while signing. responder_cert must be signed " + "by private_key", + errors, + ) + + return basic + + def create_ocsp_response( + self, response_status, builder, private_key, algorithm + ): + if response_status is ocsp.OCSPResponseStatus.SUCCESSFUL: + basic = self._create_ocsp_basic_response( + builder, private_key, algorithm + ) + else: + basic = self._ffi.NULL + + ocsp_resp = self._lib.OCSP_response_create( + response_status.value, basic + ) + self.openssl_assert(ocsp_resp != self._ffi.NULL) + ocsp_resp = self._ffi.gc(ocsp_resp, self._lib.OCSP_RESPONSE_free) + return _OCSPResponse(self, ocsp_resp) + + def elliptic_curve_exchange_algorithm_supported(self, algorithm, curve): + return self.elliptic_curve_supported(curve) and isinstance( + algorithm, ec.ECDH + ) + + def _ec_cdata_to_evp_pkey(self, ec_cdata): + evp_pkey = self._create_evp_pkey_gc() + res = self._lib.EVP_PKEY_set1_EC_KEY(evp_pkey, ec_cdata) + self.openssl_assert(res == 1) + return evp_pkey + + def _elliptic_curve_to_nid(self, curve): + """ + Get the NID for a curve name. + """ + + curve_aliases = {"secp192r1": "prime192v1", "secp256r1": "prime256v1"} + + curve_name = curve_aliases.get(curve.name, curve.name) + + curve_nid = self._lib.OBJ_sn2nid(curve_name.encode()) + if curve_nid == self._lib.NID_undef: + raise UnsupportedAlgorithm( + "{} is not a supported elliptic curve".format(curve.name), + _Reasons.UNSUPPORTED_ELLIPTIC_CURVE, + ) + return curve_nid + + @contextmanager + def _tmp_bn_ctx(self): + bn_ctx = self._lib.BN_CTX_new() + self.openssl_assert(bn_ctx != self._ffi.NULL) + bn_ctx = self._ffi.gc(bn_ctx, self._lib.BN_CTX_free) + self._lib.BN_CTX_start(bn_ctx) + try: + yield bn_ctx + finally: + self._lib.BN_CTX_end(bn_ctx) + + def _ec_key_determine_group_get_func(self, ctx): + """ + Given an EC_KEY determine the group and what function is required to + get point coordinates. + """ + self.openssl_assert(ctx != self._ffi.NULL) + + nid_two_field = self._lib.OBJ_sn2nid(b"characteristic-two-field") + self.openssl_assert(nid_two_field != self._lib.NID_undef) + + group = self._lib.EC_KEY_get0_group(ctx) + self.openssl_assert(group != self._ffi.NULL) + + method = self._lib.EC_GROUP_method_of(group) + self.openssl_assert(method != self._ffi.NULL) + + nid = self._lib.EC_METHOD_get_field_type(method) + self.openssl_assert(nid != self._lib.NID_undef) + + if nid == nid_two_field and self._lib.Cryptography_HAS_EC2M: + get_func = self._lib.EC_POINT_get_affine_coordinates_GF2m + else: + get_func = self._lib.EC_POINT_get_affine_coordinates_GFp + + assert get_func + + return get_func, group + + def _ec_key_set_public_key_affine_coordinates(self, ctx, x, y): + """ + Sets the public key point in the EC_KEY context to the affine x and y + values. + """ + + if x < 0 or y < 0: + raise ValueError( + "Invalid EC key. Both x and y must be non-negative." + ) + + x = self._ffi.gc(self._int_to_bn(x), self._lib.BN_free) + y = self._ffi.gc(self._int_to_bn(y), self._lib.BN_free) + res = self._lib.EC_KEY_set_public_key_affine_coordinates(ctx, x, y) + if res != 1: + self._consume_errors() + raise ValueError("Invalid EC key.") + + return ctx + + def _private_key_bytes( + self, encoding, format, encryption_algorithm, key, evp_pkey, cdata + ): + # validate argument types + if not isinstance(encoding, serialization.Encoding): + raise TypeError("encoding must be an item from the Encoding enum") + if not isinstance(format, serialization.PrivateFormat): + raise TypeError( + "format must be an item from the PrivateFormat enum" + ) + if not isinstance( + encryption_algorithm, serialization.KeySerializationEncryption + ): + raise TypeError( + "Encryption algorithm must be a KeySerializationEncryption " + "instance" + ) + + # validate password + if isinstance(encryption_algorithm, serialization.NoEncryption): + password = b"" + elif isinstance( + encryption_algorithm, serialization.BestAvailableEncryption + ): + password = encryption_algorithm.password + if len(password) > 1023: + raise ValueError( + "Passwords longer than 1023 bytes are not supported by " + "this backend" + ) + else: + raise ValueError("Unsupported encryption type") + + # PKCS8 + PEM/DER + if format is serialization.PrivateFormat.PKCS8: + if encoding is serialization.Encoding.PEM: + write_bio = self._lib.PEM_write_bio_PKCS8PrivateKey + elif encoding is serialization.Encoding.DER: + write_bio = self._lib.i2d_PKCS8PrivateKey_bio + else: + raise ValueError("Unsupported encoding for PKCS8") + return self._private_key_bytes_via_bio( + write_bio, evp_pkey, password + ) + + # TraditionalOpenSSL + PEM/DER + if format is serialization.PrivateFormat.TraditionalOpenSSL: + if self._fips_enabled and not isinstance( + encryption_algorithm, serialization.NoEncryption + ): + raise ValueError( + "Encrypted traditional OpenSSL format is not " + "supported in FIPS mode." + ) + key_type = self._lib.EVP_PKEY_id(evp_pkey) + + if encoding is serialization.Encoding.PEM: + if key_type == self._lib.EVP_PKEY_RSA: + write_bio = self._lib.PEM_write_bio_RSAPrivateKey + elif key_type == self._lib.EVP_PKEY_DSA: + write_bio = self._lib.PEM_write_bio_DSAPrivateKey + elif key_type == self._lib.EVP_PKEY_EC: + write_bio = self._lib.PEM_write_bio_ECPrivateKey + else: + raise ValueError( + "Unsupported key type for TraditionalOpenSSL" + ) + return self._private_key_bytes_via_bio( + write_bio, cdata, password + ) + + if encoding is serialization.Encoding.DER: + if password: + raise ValueError( + "Encryption is not supported for DER encoded " + "traditional OpenSSL keys" + ) + if key_type == self._lib.EVP_PKEY_RSA: + write_bio = self._lib.i2d_RSAPrivateKey_bio + elif key_type == self._lib.EVP_PKEY_EC: + write_bio = self._lib.i2d_ECPrivateKey_bio + elif key_type == self._lib.EVP_PKEY_DSA: + write_bio = self._lib.i2d_DSAPrivateKey_bio + else: + raise ValueError( + "Unsupported key type for TraditionalOpenSSL" + ) + return self._bio_func_output(write_bio, cdata) + + raise ValueError("Unsupported encoding for TraditionalOpenSSL") + + # OpenSSH + PEM + if format is serialization.PrivateFormat.OpenSSH: + if encoding is serialization.Encoding.PEM: + return ssh.serialize_ssh_private_key(key, password) + + raise ValueError( + "OpenSSH private key format can only be used" + " with PEM encoding" + ) + + # Anything that key-specific code was supposed to handle earlier, + # like Raw. + raise ValueError("format is invalid with this key") + + def _private_key_bytes_via_bio(self, write_bio, evp_pkey, password): + if not password: + evp_cipher = self._ffi.NULL + else: + # This is a curated value that we will update over time. + evp_cipher = self._lib.EVP_get_cipherbyname(b"aes-256-cbc") + + return self._bio_func_output( + write_bio, + evp_pkey, + evp_cipher, + password, + len(password), + self._ffi.NULL, + self._ffi.NULL, + ) + + def _bio_func_output(self, write_bio, *args): + bio = self._create_mem_bio_gc() + res = write_bio(bio, *args) + self.openssl_assert(res == 1) + return self._read_mem_bio(bio) + + def _public_key_bytes(self, encoding, format, key, evp_pkey, cdata): + if not isinstance(encoding, serialization.Encoding): + raise TypeError("encoding must be an item from the Encoding enum") + if not isinstance(format, serialization.PublicFormat): + raise TypeError( + "format must be an item from the PublicFormat enum" + ) + + # SubjectPublicKeyInfo + PEM/DER + if format is serialization.PublicFormat.SubjectPublicKeyInfo: + if encoding is serialization.Encoding.PEM: + write_bio = self._lib.PEM_write_bio_PUBKEY + elif encoding is serialization.Encoding.DER: + write_bio = self._lib.i2d_PUBKEY_bio + else: + raise ValueError( + "SubjectPublicKeyInfo works only with PEM or DER encoding" + ) + return self._bio_func_output(write_bio, evp_pkey) + + # PKCS1 + PEM/DER + if format is serialization.PublicFormat.PKCS1: + # Only RSA is supported here. + key_type = self._lib.EVP_PKEY_id(evp_pkey) + if key_type != self._lib.EVP_PKEY_RSA: + raise ValueError("PKCS1 format is supported only for RSA keys") + + if encoding is serialization.Encoding.PEM: + write_bio = self._lib.PEM_write_bio_RSAPublicKey + elif encoding is serialization.Encoding.DER: + write_bio = self._lib.i2d_RSAPublicKey_bio + else: + raise ValueError("PKCS1 works only with PEM or DER encoding") + return self._bio_func_output(write_bio, cdata) + + # OpenSSH + OpenSSH + if format is serialization.PublicFormat.OpenSSH: + if encoding is serialization.Encoding.OpenSSH: + return ssh.serialize_ssh_public_key(key) + + raise ValueError( + "OpenSSH format must be used with OpenSSH encoding" + ) + + # Anything that key-specific code was supposed to handle earlier, + # like Raw, CompressedPoint, UncompressedPoint + raise ValueError("format is invalid with this key") + + def _parameter_bytes(self, encoding, format, cdata): + if encoding is serialization.Encoding.OpenSSH: + raise TypeError("OpenSSH encoding is not supported") + + # Only DH is supported here currently. + q = self._ffi.new("BIGNUM **") + self._lib.DH_get0_pqg(cdata, self._ffi.NULL, q, self._ffi.NULL) + if encoding is serialization.Encoding.PEM: + if q[0] != self._ffi.NULL: + write_bio = self._lib.PEM_write_bio_DHxparams + else: + write_bio = self._lib.PEM_write_bio_DHparams + elif encoding is serialization.Encoding.DER: + if q[0] != self._ffi.NULL: + write_bio = self._lib.Cryptography_i2d_DHxparams_bio + else: + write_bio = self._lib.i2d_DHparams_bio + else: + raise TypeError("encoding must be an item from the Encoding enum") + + bio = self._create_mem_bio_gc() + res = write_bio(bio, cdata) + self.openssl_assert(res == 1) + return self._read_mem_bio(bio) + + def generate_dh_parameters(self, generator, key_size): + if key_size < dh._MIN_MODULUS_SIZE: + raise ValueError( + "DH key_size must be at least {} bits".format( + dh._MIN_MODULUS_SIZE + ) + ) + + if generator not in (2, 5): + raise ValueError("DH generator must be 2 or 5") + + dh_param_cdata = self._lib.DH_new() + self.openssl_assert(dh_param_cdata != self._ffi.NULL) + dh_param_cdata = self._ffi.gc(dh_param_cdata, self._lib.DH_free) + + res = self._lib.DH_generate_parameters_ex( + dh_param_cdata, key_size, generator, self._ffi.NULL + ) + self.openssl_assert(res == 1) + + return _DHParameters(self, dh_param_cdata) + + def _dh_cdata_to_evp_pkey(self, dh_cdata): + evp_pkey = self._create_evp_pkey_gc() + res = self._lib.EVP_PKEY_set1_DH(evp_pkey, dh_cdata) + self.openssl_assert(res == 1) + return evp_pkey + + def generate_dh_private_key(self, parameters): + dh_key_cdata = _dh_params_dup(parameters._dh_cdata, self) + + res = self._lib.DH_generate_key(dh_key_cdata) + self.openssl_assert(res == 1) + + evp_pkey = self._dh_cdata_to_evp_pkey(dh_key_cdata) + + return _DHPrivateKey(self, dh_key_cdata, evp_pkey) + + def generate_dh_private_key_and_parameters(self, generator, key_size): + return self.generate_dh_private_key( + self.generate_dh_parameters(generator, key_size) + ) + + def load_dh_private_numbers(self, numbers): + parameter_numbers = numbers.public_numbers.parameter_numbers + + dh_cdata = self._lib.DH_new() + self.openssl_assert(dh_cdata != self._ffi.NULL) + dh_cdata = self._ffi.gc(dh_cdata, self._lib.DH_free) + + p = self._int_to_bn(parameter_numbers.p) + g = self._int_to_bn(parameter_numbers.g) + + if parameter_numbers.q is not None: + q = self._int_to_bn(parameter_numbers.q) + else: + q = self._ffi.NULL + + pub_key = self._int_to_bn(numbers.public_numbers.y) + priv_key = self._int_to_bn(numbers.x) + + res = self._lib.DH_set0_pqg(dh_cdata, p, q, g) + self.openssl_assert(res == 1) + + res = self._lib.DH_set0_key(dh_cdata, pub_key, priv_key) + self.openssl_assert(res == 1) + + codes = self._ffi.new("int[]", 1) + res = self._lib.Cryptography_DH_check(dh_cdata, codes) + self.openssl_assert(res == 1) + + # DH_check will return DH_NOT_SUITABLE_GENERATOR if p % 24 does not + # equal 11 when the generator is 2 (a quadratic nonresidue). + # We want to ignore that error because p % 24 == 23 is also fine. + # Specifically, g is then a quadratic residue. Within the context of + # Diffie-Hellman this means it can only generate half the possible + # values. That sounds bad, but quadratic nonresidues leak a bit of + # the key to the attacker in exchange for having the full key space + # available. See: https://crypto.stackexchange.com/questions/12961 + if codes[0] != 0 and not ( + parameter_numbers.g == 2 + and codes[0] ^ self._lib.DH_NOT_SUITABLE_GENERATOR == 0 + ): + raise ValueError("DH private numbers did not pass safety checks.") + + evp_pkey = self._dh_cdata_to_evp_pkey(dh_cdata) + + return _DHPrivateKey(self, dh_cdata, evp_pkey) + + def load_dh_public_numbers(self, numbers): + dh_cdata = self._lib.DH_new() + self.openssl_assert(dh_cdata != self._ffi.NULL) + dh_cdata = self._ffi.gc(dh_cdata, self._lib.DH_free) + + parameter_numbers = numbers.parameter_numbers + + p = self._int_to_bn(parameter_numbers.p) + g = self._int_to_bn(parameter_numbers.g) + + if parameter_numbers.q is not None: + q = self._int_to_bn(parameter_numbers.q) + else: + q = self._ffi.NULL + + pub_key = self._int_to_bn(numbers.y) + + res = self._lib.DH_set0_pqg(dh_cdata, p, q, g) + self.openssl_assert(res == 1) + + res = self._lib.DH_set0_key(dh_cdata, pub_key, self._ffi.NULL) + self.openssl_assert(res == 1) + + evp_pkey = self._dh_cdata_to_evp_pkey(dh_cdata) + + return _DHPublicKey(self, dh_cdata, evp_pkey) + + def load_dh_parameter_numbers(self, numbers): + dh_cdata = self._lib.DH_new() + self.openssl_assert(dh_cdata != self._ffi.NULL) + dh_cdata = self._ffi.gc(dh_cdata, self._lib.DH_free) + + p = self._int_to_bn(numbers.p) + g = self._int_to_bn(numbers.g) + + if numbers.q is not None: + q = self._int_to_bn(numbers.q) + else: + q = self._ffi.NULL + + res = self._lib.DH_set0_pqg(dh_cdata, p, q, g) + self.openssl_assert(res == 1) + + return _DHParameters(self, dh_cdata) + + def dh_parameters_supported(self, p, g, q=None): + dh_cdata = self._lib.DH_new() + self.openssl_assert(dh_cdata != self._ffi.NULL) + dh_cdata = self._ffi.gc(dh_cdata, self._lib.DH_free) + + p = self._int_to_bn(p) + g = self._int_to_bn(g) + + if q is not None: + q = self._int_to_bn(q) + else: + q = self._ffi.NULL + + res = self._lib.DH_set0_pqg(dh_cdata, p, q, g) + self.openssl_assert(res == 1) + + codes = self._ffi.new("int[]", 1) + res = self._lib.Cryptography_DH_check(dh_cdata, codes) + self.openssl_assert(res == 1) + + return codes[0] == 0 + + def dh_x942_serialization_supported(self): + return self._lib.Cryptography_HAS_EVP_PKEY_DHX == 1 + + def x509_name_bytes(self, name): + x509_name = _encode_name_gc(self, name) + pp = self._ffi.new("unsigned char **") + res = self._lib.i2d_X509_NAME(x509_name, pp) + self.openssl_assert(pp[0] != self._ffi.NULL) + pp = self._ffi.gc( + pp, lambda pointer: self._lib.OPENSSL_free(pointer[0]) + ) + self.openssl_assert(res > 0) + return self._ffi.buffer(pp[0], res)[:] + + def x25519_load_public_bytes(self, data): + # When we drop support for CRYPTOGRAPHY_OPENSSL_LESS_THAN_111 we can + # switch this to EVP_PKEY_new_raw_public_key + if len(data) != 32: + raise ValueError("An X25519 public key is 32 bytes long") + + evp_pkey = self._create_evp_pkey_gc() + res = self._lib.EVP_PKEY_set_type(evp_pkey, self._lib.NID_X25519) + self.openssl_assert(res == 1) + res = self._lib.EVP_PKEY_set1_tls_encodedpoint( + evp_pkey, data, len(data) + ) + self.openssl_assert(res == 1) + return _X25519PublicKey(self, evp_pkey) + + def x25519_load_private_bytes(self, data): + # When we drop support for CRYPTOGRAPHY_OPENSSL_LESS_THAN_111 we can + # switch this to EVP_PKEY_new_raw_private_key and drop the + # zeroed_bytearray garbage. + # OpenSSL only has facilities for loading PKCS8 formatted private + # keys using the algorithm identifiers specified in + # https://tools.ietf.org/html/draft-ietf-curdle-pkix-09. + # This is the standard PKCS8 prefix for a 32 byte X25519 key. + # The form is: + # 0:d=0 hl=2 l= 46 cons: SEQUENCE + # 2:d=1 hl=2 l= 1 prim: INTEGER :00 + # 5:d=1 hl=2 l= 5 cons: SEQUENCE + # 7:d=2 hl=2 l= 3 prim: OBJECT :1.3.101.110 + # 12:d=1 hl=2 l= 34 prim: OCTET STRING (the key) + # Of course there's a bit more complexity. In reality OCTET STRING + # contains an OCTET STRING of length 32! So the last two bytes here + # are \x04\x20, which is an OCTET STRING of length 32. + if len(data) != 32: + raise ValueError("An X25519 private key is 32 bytes long") + + pkcs8_prefix = b'0.\x02\x01\x000\x05\x06\x03+en\x04"\x04 ' + with self._zeroed_bytearray(48) as ba: + ba[0:16] = pkcs8_prefix + ba[16:] = data + bio = self._bytes_to_bio(ba) + evp_pkey = self._lib.d2i_PrivateKey_bio(bio.bio, self._ffi.NULL) + + self.openssl_assert(evp_pkey != self._ffi.NULL) + evp_pkey = self._ffi.gc(evp_pkey, self._lib.EVP_PKEY_free) + self.openssl_assert( + self._lib.EVP_PKEY_id(evp_pkey) == self._lib.EVP_PKEY_X25519 + ) + return _X25519PrivateKey(self, evp_pkey) + + def _evp_pkey_keygen_gc(self, nid): + evp_pkey_ctx = self._lib.EVP_PKEY_CTX_new_id(nid, self._ffi.NULL) + self.openssl_assert(evp_pkey_ctx != self._ffi.NULL) + evp_pkey_ctx = self._ffi.gc(evp_pkey_ctx, self._lib.EVP_PKEY_CTX_free) + res = self._lib.EVP_PKEY_keygen_init(evp_pkey_ctx) + self.openssl_assert(res == 1) + evp_ppkey = self._ffi.new("EVP_PKEY **") + res = self._lib.EVP_PKEY_keygen(evp_pkey_ctx, evp_ppkey) + self.openssl_assert(res == 1) + self.openssl_assert(evp_ppkey[0] != self._ffi.NULL) + evp_pkey = self._ffi.gc(evp_ppkey[0], self._lib.EVP_PKEY_free) + return evp_pkey + + def x25519_generate_key(self): + evp_pkey = self._evp_pkey_keygen_gc(self._lib.NID_X25519) + return _X25519PrivateKey(self, evp_pkey) + + def x25519_supported(self): + if self._fips_enabled: + return False + return not self._lib.CRYPTOGRAPHY_IS_LIBRESSL + + def x448_load_public_bytes(self, data): + if len(data) != 56: + raise ValueError("An X448 public key is 56 bytes long") + + evp_pkey = self._lib.EVP_PKEY_new_raw_public_key( + self._lib.NID_X448, self._ffi.NULL, data, len(data) + ) + self.openssl_assert(evp_pkey != self._ffi.NULL) + evp_pkey = self._ffi.gc(evp_pkey, self._lib.EVP_PKEY_free) + return _X448PublicKey(self, evp_pkey) + + def x448_load_private_bytes(self, data): + if len(data) != 56: + raise ValueError("An X448 private key is 56 bytes long") + + data_ptr = self._ffi.from_buffer(data) + evp_pkey = self._lib.EVP_PKEY_new_raw_private_key( + self._lib.NID_X448, self._ffi.NULL, data_ptr, len(data) + ) + self.openssl_assert(evp_pkey != self._ffi.NULL) + evp_pkey = self._ffi.gc(evp_pkey, self._lib.EVP_PKEY_free) + return _X448PrivateKey(self, evp_pkey) + + def x448_generate_key(self): + evp_pkey = self._evp_pkey_keygen_gc(self._lib.NID_X448) + return _X448PrivateKey(self, evp_pkey) + + def x448_supported(self): + if self._fips_enabled: + return False + return not self._lib.CRYPTOGRAPHY_OPENSSL_LESS_THAN_111 + + def ed25519_supported(self): + if self._fips_enabled: + return False + return not self._lib.CRYPTOGRAPHY_OPENSSL_LESS_THAN_111B + + def ed25519_load_public_bytes(self, data): + utils._check_bytes("data", data) + + if len(data) != ed25519._ED25519_KEY_SIZE: + raise ValueError("An Ed25519 public key is 32 bytes long") + + evp_pkey = self._lib.EVP_PKEY_new_raw_public_key( + self._lib.NID_ED25519, self._ffi.NULL, data, len(data) + ) + self.openssl_assert(evp_pkey != self._ffi.NULL) + evp_pkey = self._ffi.gc(evp_pkey, self._lib.EVP_PKEY_free) + + return _Ed25519PublicKey(self, evp_pkey) + + def ed25519_load_private_bytes(self, data): + if len(data) != ed25519._ED25519_KEY_SIZE: + raise ValueError("An Ed25519 private key is 32 bytes long") + + utils._check_byteslike("data", data) + data_ptr = self._ffi.from_buffer(data) + evp_pkey = self._lib.EVP_PKEY_new_raw_private_key( + self._lib.NID_ED25519, self._ffi.NULL, data_ptr, len(data) + ) + self.openssl_assert(evp_pkey != self._ffi.NULL) + evp_pkey = self._ffi.gc(evp_pkey, self._lib.EVP_PKEY_free) + + return _Ed25519PrivateKey(self, evp_pkey) + + def ed25519_generate_key(self): + evp_pkey = self._evp_pkey_keygen_gc(self._lib.NID_ED25519) + return _Ed25519PrivateKey(self, evp_pkey) + + def ed448_supported(self): + if self._fips_enabled: + return False + return not self._lib.CRYPTOGRAPHY_OPENSSL_LESS_THAN_111B + + def ed448_load_public_bytes(self, data): + utils._check_bytes("data", data) + if len(data) != _ED448_KEY_SIZE: + raise ValueError("An Ed448 public key is 57 bytes long") + + evp_pkey = self._lib.EVP_PKEY_new_raw_public_key( + self._lib.NID_ED448, self._ffi.NULL, data, len(data) + ) + self.openssl_assert(evp_pkey != self._ffi.NULL) + evp_pkey = self._ffi.gc(evp_pkey, self._lib.EVP_PKEY_free) + + return _Ed448PublicKey(self, evp_pkey) + + def ed448_load_private_bytes(self, data): + utils._check_byteslike("data", data) + if len(data) != _ED448_KEY_SIZE: + raise ValueError("An Ed448 private key is 57 bytes long") + + data_ptr = self._ffi.from_buffer(data) + evp_pkey = self._lib.EVP_PKEY_new_raw_private_key( + self._lib.NID_ED448, self._ffi.NULL, data_ptr, len(data) + ) + self.openssl_assert(evp_pkey != self._ffi.NULL) + evp_pkey = self._ffi.gc(evp_pkey, self._lib.EVP_PKEY_free) + + return _Ed448PrivateKey(self, evp_pkey) + + def ed448_generate_key(self): + evp_pkey = self._evp_pkey_keygen_gc(self._lib.NID_ED448) + return _Ed448PrivateKey(self, evp_pkey) + + def derive_scrypt(self, key_material, salt, length, n, r, p): + buf = self._ffi.new("unsigned char[]", length) + key_material_ptr = self._ffi.from_buffer(key_material) + res = self._lib.EVP_PBE_scrypt( + key_material_ptr, + len(key_material), + salt, + len(salt), + n, + r, + p, + scrypt._MEM_LIMIT, + buf, + length, + ) + if res != 1: + errors = self._consume_errors_with_text() + # memory required formula explained here: + # https://blog.filippo.io/the-scrypt-parameters/ + min_memory = 128 * n * r // (1024 ** 2) + raise MemoryError( + "Not enough memory to derive key. These parameters require" + " {} MB of memory.".format(min_memory), + errors, + ) + return self._ffi.buffer(buf)[:] + + def aead_cipher_supported(self, cipher): + cipher_name = aead._aead_cipher_name(cipher) + if self._fips_enabled and cipher_name not in self._fips_aead: + return False + return self._lib.EVP_get_cipherbyname(cipher_name) != self._ffi.NULL + + @contextlib.contextmanager + def _zeroed_bytearray(self, length): + """ + This method creates a bytearray, which we copy data into (hopefully + also from a mutable buffer that can be dynamically erased!), and then + zero when we're done. + """ + ba = bytearray(length) + try: + yield ba + finally: + self._zero_data(ba, length) + + def _zero_data(self, data, length): + # We clear things this way because at the moment we're not + # sure of a better way that can guarantee it overwrites the + # memory of a bytearray and doesn't just replace the underlying char *. + for i in range(length): + data[i] = 0 + + @contextlib.contextmanager + def _zeroed_null_terminated_buf(self, data): + """ + This method takes bytes, which can be a bytestring or a mutable + buffer like a bytearray, and yields a null-terminated version of that + data. This is required because PKCS12_parse doesn't take a length with + its password char * and ffi.from_buffer doesn't provide null + termination. So, to support zeroing the data via bytearray we + need to build this ridiculous construct that copies the memory, but + zeroes it after use. + """ + if data is None: + yield self._ffi.NULL + else: + data_len = len(data) + buf = self._ffi.new("char[]", data_len + 1) + self._ffi.memmove(buf, data, data_len) + try: + yield buf + finally: + # Cast to a uint8_t * so we can assign by integer + self._zero_data(self._ffi.cast("uint8_t *", buf), data_len) + + def load_key_and_certificates_from_pkcs12(self, data, password): + if password is not None: + utils._check_byteslike("password", password) + + bio = self._bytes_to_bio(data) + p12 = self._lib.d2i_PKCS12_bio(bio.bio, self._ffi.NULL) + if p12 == self._ffi.NULL: + self._consume_errors() + raise ValueError("Could not deserialize PKCS12 data") + + p12 = self._ffi.gc(p12, self._lib.PKCS12_free) + evp_pkey_ptr = self._ffi.new("EVP_PKEY **") + x509_ptr = self._ffi.new("X509 **") + sk_x509_ptr = self._ffi.new("Cryptography_STACK_OF_X509 **") + with self._zeroed_null_terminated_buf(password) as password_buf: + res = self._lib.PKCS12_parse( + p12, password_buf, evp_pkey_ptr, x509_ptr, sk_x509_ptr + ) + + if res == 0: + self._consume_errors() + raise ValueError("Invalid password or PKCS12 data") + + cert = None + key = None + additional_certificates = [] + + if evp_pkey_ptr[0] != self._ffi.NULL: + evp_pkey = self._ffi.gc(evp_pkey_ptr[0], self._lib.EVP_PKEY_free) + key = self._evp_pkey_to_private_key(evp_pkey) + + if x509_ptr[0] != self._ffi.NULL: + x509 = self._ffi.gc(x509_ptr[0], self._lib.X509_free) + cert = _Certificate(self, x509) + + if sk_x509_ptr[0] != self._ffi.NULL: + sk_x509 = self._ffi.gc(sk_x509_ptr[0], self._lib.sk_X509_free) + num = self._lib.sk_X509_num(sk_x509_ptr[0]) + for i in range(num): + x509 = self._lib.sk_X509_value(sk_x509, i) + self.openssl_assert(x509 != self._ffi.NULL) + x509 = self._ffi.gc(x509, self._lib.X509_free) + additional_certificates.append(_Certificate(self, x509)) + + return (key, cert, additional_certificates) + + def serialize_key_and_certificates_to_pkcs12( + self, name, key, cert, cas, encryption_algorithm + ): + password = None + if name is not None: + utils._check_bytes("name", name) + + if isinstance(encryption_algorithm, serialization.NoEncryption): + nid_cert = -1 + nid_key = -1 + pkcs12_iter = 0 + mac_iter = 0 + elif isinstance( + encryption_algorithm, serialization.BestAvailableEncryption + ): + # PKCS12 encryption is hopeless trash and can never be fixed. + # This is the least terrible option. + nid_cert = self._lib.NID_pbe_WithSHA1And3_Key_TripleDES_CBC + nid_key = self._lib.NID_pbe_WithSHA1And3_Key_TripleDES_CBC + # At least we can set this higher than OpenSSL's default + pkcs12_iter = 20000 + # mac_iter chosen for compatibility reasons, see: + # https://www.openssl.org/docs/man1.1.1/man3/PKCS12_create.html + # Did we mention how lousy PKCS12 encryption is? + mac_iter = 1 + password = encryption_algorithm.password + else: + raise ValueError("Unsupported key encryption type") + + if cas is None or len(cas) == 0: + sk_x509 = self._ffi.NULL + else: + sk_x509 = self._lib.sk_X509_new_null() + sk_x509 = self._ffi.gc(sk_x509, self._lib.sk_X509_free) + + # reverse the list when building the stack so that they're encoded + # in the order they were originally provided. it is a mystery + for ca in reversed(cas): + res = self._lib.sk_X509_push(sk_x509, ca._x509) + backend.openssl_assert(res >= 1) + + with self._zeroed_null_terminated_buf(password) as password_buf: + with self._zeroed_null_terminated_buf(name) as name_buf: + p12 = self._lib.PKCS12_create( + password_buf, + name_buf, + key._evp_pkey if key else self._ffi.NULL, + cert._x509 if cert else self._ffi.NULL, + sk_x509, + nid_key, + nid_cert, + pkcs12_iter, + mac_iter, + 0, + ) + + self.openssl_assert(p12 != self._ffi.NULL) + p12 = self._ffi.gc(p12, self._lib.PKCS12_free) + + bio = self._create_mem_bio_gc() + res = self._lib.i2d_PKCS12_bio(bio, p12) + self.openssl_assert(res > 0) + return self._read_mem_bio(bio) + + def poly1305_supported(self): + if self._fips_enabled: + return False + return self._lib.Cryptography_HAS_POLY1305 == 1 + + def create_poly1305_ctx(self, key): + utils._check_byteslike("key", key) + if len(key) != _POLY1305_KEY_SIZE: + raise ValueError("A poly1305 key is 32 bytes long") + + return _Poly1305Context(self, key) + + def load_pem_pkcs7_certificates(self, data): + utils._check_bytes("data", data) + bio = self._bytes_to_bio(data) + p7 = self._lib.PEM_read_bio_PKCS7( + bio.bio, self._ffi.NULL, self._ffi.NULL, self._ffi.NULL + ) + if p7 == self._ffi.NULL: + self._consume_errors() + raise ValueError("Unable to parse PKCS7 data") + + p7 = self._ffi.gc(p7, self._lib.PKCS7_free) + return self._load_pkcs7_certificates(p7) + + def load_der_pkcs7_certificates(self, data): + utils._check_bytes("data", data) + bio = self._bytes_to_bio(data) + p7 = self._lib.d2i_PKCS7_bio(bio.bio, self._ffi.NULL) + if p7 == self._ffi.NULL: + self._consume_errors() + raise ValueError("Unable to parse PKCS7 data") + + p7 = self._ffi.gc(p7, self._lib.PKCS7_free) + return self._load_pkcs7_certificates(p7) + + def _load_pkcs7_certificates(self, p7): + nid = self._lib.OBJ_obj2nid(p7.type) + self.openssl_assert(nid != self._lib.NID_undef) + if nid != self._lib.NID_pkcs7_signed: + raise UnsupportedAlgorithm( + "Only basic signed structures are currently supported. NID" + " for this data was {}".format(nid), + _Reasons.UNSUPPORTED_SERIALIZATION, + ) + + sk_x509 = p7.d.sign.cert + num = self._lib.sk_X509_num(sk_x509) + certs = [] + for i in range(num): + x509 = self._lib.sk_X509_value(sk_x509, i) + self.openssl_assert(x509 != self._ffi.NULL) + res = self._lib.X509_up_ref(x509) + # When OpenSSL is less than 1.1.0 up_ref returns the current + # refcount. On 1.1.0+ it returns 1 for success. + self.openssl_assert(res >= 1) + x509 = self._ffi.gc(x509, self._lib.X509_free) + certs.append(_Certificate(self, x509)) + + return certs + + def pkcs7_sign(self, builder, encoding, options): + bio = self._bytes_to_bio(builder._data) + init_flags = self._lib.PKCS7_PARTIAL + final_flags = 0 + + if len(builder._additional_certs) == 0: + certs = self._ffi.NULL + else: + certs = self._lib.sk_X509_new_null() + certs = self._ffi.gc(certs, self._lib.sk_X509_free) + for cert in builder._additional_certs: + res = self._lib.sk_X509_push(certs, cert._x509) + self.openssl_assert(res >= 1) + + if pkcs7.PKCS7Options.DetachedSignature in options: + # Don't embed the data in the PKCS7 structure + init_flags |= self._lib.PKCS7_DETACHED + final_flags |= self._lib.PKCS7_DETACHED + + # This just inits a structure for us. However, there + # are flags we need to set, joy. + p7 = self._lib.PKCS7_sign( + self._ffi.NULL, + self._ffi.NULL, + certs, + self._ffi.NULL, + init_flags, + ) + self.openssl_assert(p7 != self._ffi.NULL) + p7 = self._ffi.gc(p7, self._lib.PKCS7_free) + signer_flags = 0 + # These flags are configurable on a per-signature basis + # but we've deliberately chosen to make the API only allow + # setting it across all signatures for now. + if pkcs7.PKCS7Options.NoCapabilities in options: + signer_flags |= self._lib.PKCS7_NOSMIMECAP + elif pkcs7.PKCS7Options.NoAttributes in options: + signer_flags |= self._lib.PKCS7_NOATTR + + if pkcs7.PKCS7Options.NoCerts in options: + signer_flags |= self._lib.PKCS7_NOCERTS + + for certificate, private_key, hash_algorithm in builder._signers: + md = self._evp_md_non_null_from_algorithm(hash_algorithm) + p7signerinfo = self._lib.PKCS7_sign_add_signer( + p7, certificate._x509, private_key._evp_pkey, md, signer_flags + ) + self.openssl_assert(p7signerinfo != self._ffi.NULL) + + for option in options: + # DetachedSignature, NoCapabilities, and NoAttributes are already + # handled so we just need to check these last two options. + if option is pkcs7.PKCS7Options.Text: + final_flags |= self._lib.PKCS7_TEXT + elif option is pkcs7.PKCS7Options.Binary: + final_flags |= self._lib.PKCS7_BINARY + + bio_out = self._create_mem_bio_gc() + if encoding is serialization.Encoding.SMIME: + # This finalizes the structure + res = self._lib.SMIME_write_PKCS7( + bio_out, p7, bio.bio, final_flags + ) + elif encoding is serialization.Encoding.PEM: + res = self._lib.PKCS7_final(p7, bio.bio, final_flags) + self.openssl_assert(res == 1) + res = self._lib.PEM_write_bio_PKCS7_stream( + bio_out, p7, bio.bio, final_flags + ) + else: + assert encoding is serialization.Encoding.DER + # We need to call finalize here becauase i2d_PKCS7_bio does not + # finalize. + res = self._lib.PKCS7_final(p7, bio.bio, final_flags) + self.openssl_assert(res == 1) + res = self._lib.i2d_PKCS7_bio(bio_out, p7) + self.openssl_assert(res == 1) + return self._read_mem_bio(bio_out) + + +class GetCipherByName(object): + def __init__(self, fmt): + self._fmt = fmt + + def __call__(self, backend, cipher, mode): + cipher_name = self._fmt.format(cipher=cipher, mode=mode).lower() + return backend._lib.EVP_get_cipherbyname(cipher_name.encode("ascii")) + + +def _get_xts_cipher(backend, cipher, mode): + cipher_name = "aes-{}-xts".format(cipher.key_size // 2) + return backend._lib.EVP_get_cipherbyname(cipher_name.encode("ascii")) + + +backend = Backend() diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/openssl/ciphers.py b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/openssl/ciphers.py new file mode 100644 index 0000000..1e805d2 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/openssl/ciphers.py @@ -0,0 +1,231 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function + +from cryptography import utils +from cryptography.exceptions import InvalidTag, UnsupportedAlgorithm, _Reasons +from cryptography.hazmat.primitives import ciphers +from cryptography.hazmat.primitives.ciphers import modes + + +@utils.register_interface(ciphers.CipherContext) +@utils.register_interface(ciphers.AEADCipherContext) +@utils.register_interface(ciphers.AEADEncryptionContext) +@utils.register_interface(ciphers.AEADDecryptionContext) +class _CipherContext(object): + _ENCRYPT = 1 + _DECRYPT = 0 + _MAX_CHUNK_SIZE = 2 ** 31 - 1 + + def __init__(self, backend, cipher, mode, operation): + self._backend = backend + self._cipher = cipher + self._mode = mode + self._operation = operation + self._tag = None + + if isinstance(self._cipher, ciphers.BlockCipherAlgorithm): + self._block_size_bytes = self._cipher.block_size // 8 + else: + self._block_size_bytes = 1 + + ctx = self._backend._lib.EVP_CIPHER_CTX_new() + ctx = self._backend._ffi.gc( + ctx, self._backend._lib.EVP_CIPHER_CTX_free + ) + + registry = self._backend._cipher_registry + try: + adapter = registry[type(cipher), type(mode)] + except KeyError: + raise UnsupportedAlgorithm( + "cipher {} in {} mode is not supported " + "by this backend.".format( + cipher.name, mode.name if mode else mode + ), + _Reasons.UNSUPPORTED_CIPHER, + ) + + evp_cipher = adapter(self._backend, cipher, mode) + if evp_cipher == self._backend._ffi.NULL: + msg = "cipher {0.name} ".format(cipher) + if mode is not None: + msg += "in {0.name} mode ".format(mode) + msg += ( + "is not supported by this backend (Your version of OpenSSL " + "may be too old. Current version: {}.)" + ).format(self._backend.openssl_version_text()) + raise UnsupportedAlgorithm(msg, _Reasons.UNSUPPORTED_CIPHER) + + if isinstance(mode, modes.ModeWithInitializationVector): + iv_nonce = self._backend._ffi.from_buffer( + mode.initialization_vector + ) + elif isinstance(mode, modes.ModeWithTweak): + iv_nonce = self._backend._ffi.from_buffer(mode.tweak) + elif isinstance(mode, modes.ModeWithNonce): + iv_nonce = self._backend._ffi.from_buffer(mode.nonce) + elif isinstance(cipher, modes.ModeWithNonce): + iv_nonce = self._backend._ffi.from_buffer(cipher.nonce) + else: + iv_nonce = self._backend._ffi.NULL + # begin init with cipher and operation type + res = self._backend._lib.EVP_CipherInit_ex( + ctx, + evp_cipher, + self._backend._ffi.NULL, + self._backend._ffi.NULL, + self._backend._ffi.NULL, + operation, + ) + self._backend.openssl_assert(res != 0) + # set the key length to handle variable key ciphers + res = self._backend._lib.EVP_CIPHER_CTX_set_key_length( + ctx, len(cipher.key) + ) + self._backend.openssl_assert(res != 0) + if isinstance(mode, modes.GCM): + res = self._backend._lib.EVP_CIPHER_CTX_ctrl( + ctx, + self._backend._lib.EVP_CTRL_AEAD_SET_IVLEN, + len(iv_nonce), + self._backend._ffi.NULL, + ) + self._backend.openssl_assert(res != 0) + if mode.tag is not None: + res = self._backend._lib.EVP_CIPHER_CTX_ctrl( + ctx, + self._backend._lib.EVP_CTRL_AEAD_SET_TAG, + len(mode.tag), + mode.tag, + ) + self._backend.openssl_assert(res != 0) + self._tag = mode.tag + + # pass key/iv + res = self._backend._lib.EVP_CipherInit_ex( + ctx, + self._backend._ffi.NULL, + self._backend._ffi.NULL, + self._backend._ffi.from_buffer(cipher.key), + iv_nonce, + operation, + ) + self._backend.openssl_assert(res != 0) + # We purposely disable padding here as it's handled higher up in the + # API. + self._backend._lib.EVP_CIPHER_CTX_set_padding(ctx, 0) + self._ctx = ctx + + def update(self, data): + buf = bytearray(len(data) + self._block_size_bytes - 1) + n = self.update_into(data, buf) + return bytes(buf[:n]) + + def update_into(self, data, buf): + total_data_len = len(data) + if len(buf) < (total_data_len + self._block_size_bytes - 1): + raise ValueError( + "buffer must be at least {} bytes for this " + "payload".format(len(data) + self._block_size_bytes - 1) + ) + + data_processed = 0 + total_out = 0 + outlen = self._backend._ffi.new("int *") + baseoutbuf = self._backend._ffi.from_buffer(buf) + baseinbuf = self._backend._ffi.from_buffer(data) + + while data_processed != total_data_len: + outbuf = baseoutbuf + total_out + inbuf = baseinbuf + data_processed + inlen = min(self._MAX_CHUNK_SIZE, total_data_len - data_processed) + + res = self._backend._lib.EVP_CipherUpdate( + self._ctx, outbuf, outlen, inbuf, inlen + ) + self._backend.openssl_assert(res != 0) + data_processed += inlen + total_out += outlen[0] + + return total_out + + def finalize(self): + if ( + self._operation == self._DECRYPT + and isinstance(self._mode, modes.ModeWithAuthenticationTag) + and self.tag is None + ): + raise ValueError( + "Authentication tag must be provided when decrypting." + ) + + buf = self._backend._ffi.new("unsigned char[]", self._block_size_bytes) + outlen = self._backend._ffi.new("int *") + res = self._backend._lib.EVP_CipherFinal_ex(self._ctx, buf, outlen) + if res == 0: + errors = self._backend._consume_errors() + + if not errors and isinstance(self._mode, modes.GCM): + raise InvalidTag + + self._backend.openssl_assert( + errors[0]._lib_reason_match( + self._backend._lib.ERR_LIB_EVP, + self._backend._lib.EVP_R_DATA_NOT_MULTIPLE_OF_BLOCK_LENGTH, + ), + errors=errors, + ) + raise ValueError( + "The length of the provided data is not a multiple of " + "the block length." + ) + + if ( + isinstance(self._mode, modes.GCM) + and self._operation == self._ENCRYPT + ): + tag_buf = self._backend._ffi.new( + "unsigned char[]", self._block_size_bytes + ) + res = self._backend._lib.EVP_CIPHER_CTX_ctrl( + self._ctx, + self._backend._lib.EVP_CTRL_AEAD_GET_TAG, + self._block_size_bytes, + tag_buf, + ) + self._backend.openssl_assert(res != 0) + self._tag = self._backend._ffi.buffer(tag_buf)[:] + + res = self._backend._lib.EVP_CIPHER_CTX_cleanup(self._ctx) + self._backend.openssl_assert(res == 1) + return self._backend._ffi.buffer(buf)[: outlen[0]] + + def finalize_with_tag(self, tag): + if len(tag) < self._mode._min_tag_length: + raise ValueError( + "Authentication tag must be {} bytes or longer.".format( + self._mode._min_tag_length + ) + ) + res = self._backend._lib.EVP_CIPHER_CTX_ctrl( + self._ctx, self._backend._lib.EVP_CTRL_AEAD_SET_TAG, len(tag), tag + ) + self._backend.openssl_assert(res != 0) + self._tag = tag + return self.finalize() + + def authenticate_additional_data(self, data): + outlen = self._backend._ffi.new("int *") + res = self._backend._lib.EVP_CipherUpdate( + self._ctx, + self._backend._ffi.NULL, + outlen, + self._backend._ffi.from_buffer(data), + len(data), + ) + self._backend.openssl_assert(res != 0) + + tag = utils.read_only_property("_tag") diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/openssl/cmac.py b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/openssl/cmac.py new file mode 100644 index 0000000..195fc23 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/openssl/cmac.py @@ -0,0 +1,82 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function + + +from cryptography import utils +from cryptography.exceptions import ( + InvalidSignature, + UnsupportedAlgorithm, + _Reasons, +) +from cryptography.hazmat.primitives import constant_time +from cryptography.hazmat.primitives.ciphers.modes import CBC + + +class _CMACContext(object): + def __init__(self, backend, algorithm, ctx=None): + if not backend.cmac_algorithm_supported(algorithm): + raise UnsupportedAlgorithm( + "This backend does not support CMAC.", + _Reasons.UNSUPPORTED_CIPHER, + ) + + self._backend = backend + self._key = algorithm.key + self._algorithm = algorithm + self._output_length = algorithm.block_size // 8 + + if ctx is None: + registry = self._backend._cipher_registry + adapter = registry[type(algorithm), CBC] + + evp_cipher = adapter(self._backend, algorithm, CBC) + + ctx = self._backend._lib.CMAC_CTX_new() + + self._backend.openssl_assert(ctx != self._backend._ffi.NULL) + ctx = self._backend._ffi.gc(ctx, self._backend._lib.CMAC_CTX_free) + + key_ptr = self._backend._ffi.from_buffer(self._key) + res = self._backend._lib.CMAC_Init( + ctx, + key_ptr, + len(self._key), + evp_cipher, + self._backend._ffi.NULL, + ) + self._backend.openssl_assert(res == 1) + + self._ctx = ctx + + algorithm = utils.read_only_property("_algorithm") + + def update(self, data): + res = self._backend._lib.CMAC_Update(self._ctx, data, len(data)) + self._backend.openssl_assert(res == 1) + + def finalize(self): + buf = self._backend._ffi.new("unsigned char[]", self._output_length) + length = self._backend._ffi.new("size_t *", self._output_length) + res = self._backend._lib.CMAC_Final(self._ctx, buf, length) + self._backend.openssl_assert(res == 1) + + self._ctx = None + + return self._backend._ffi.buffer(buf)[:] + + def copy(self): + copied_ctx = self._backend._lib.CMAC_CTX_new() + copied_ctx = self._backend._ffi.gc( + copied_ctx, self._backend._lib.CMAC_CTX_free + ) + res = self._backend._lib.CMAC_CTX_copy(copied_ctx, self._ctx) + self._backend.openssl_assert(res == 1) + return _CMACContext(self._backend, self._algorithm, ctx=copied_ctx) + + def verify(self, signature): + digest = self.finalize() + if not constant_time.bytes_eq(digest, signature): + raise InvalidSignature("Signature did not match digest.") diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/openssl/decode_asn1.py b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/openssl/decode_asn1.py new file mode 100644 index 0000000..cc9b8c0 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/openssl/decode_asn1.py @@ -0,0 +1,878 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function + +import datetime +import ipaddress + +import six + +from cryptography import x509 +from cryptography.hazmat._der import DERReader, INTEGER, NULL, SEQUENCE +from cryptography.x509.extensions import _TLS_FEATURE_TYPE_TO_ENUM +from cryptography.x509.name import _ASN1_TYPE_TO_ENUM +from cryptography.x509.oid import ( + CRLEntryExtensionOID, + CertificatePoliciesOID, + ExtensionOID, + OCSPExtensionOID, +) + + +def _obj2txt(backend, obj): + # Set to 80 on the recommendation of + # https://www.openssl.org/docs/crypto/OBJ_nid2ln.html#return_values + # + # But OIDs longer than this occur in real life (e.g. Active + # Directory makes some very long OIDs). So we need to detect + # and properly handle the case where the default buffer is not + # big enough. + # + buf_len = 80 + buf = backend._ffi.new("char[]", buf_len) + + # 'res' is the number of bytes that *would* be written if the + # buffer is large enough. If 'res' > buf_len - 1, we need to + # alloc a big-enough buffer and go again. + res = backend._lib.OBJ_obj2txt(buf, buf_len, obj, 1) + if res > buf_len - 1: # account for terminating null byte + buf_len = res + 1 + buf = backend._ffi.new("char[]", buf_len) + res = backend._lib.OBJ_obj2txt(buf, buf_len, obj, 1) + backend.openssl_assert(res > 0) + return backend._ffi.buffer(buf, res)[:].decode() + + +def _decode_x509_name_entry(backend, x509_name_entry): + obj = backend._lib.X509_NAME_ENTRY_get_object(x509_name_entry) + backend.openssl_assert(obj != backend._ffi.NULL) + data = backend._lib.X509_NAME_ENTRY_get_data(x509_name_entry) + backend.openssl_assert(data != backend._ffi.NULL) + value = _asn1_string_to_utf8(backend, data) + oid = _obj2txt(backend, obj) + type = _ASN1_TYPE_TO_ENUM[data.type] + + return x509.NameAttribute(x509.ObjectIdentifier(oid), value, type) + + +def _decode_x509_name(backend, x509_name): + count = backend._lib.X509_NAME_entry_count(x509_name) + attributes = [] + prev_set_id = -1 + for x in range(count): + entry = backend._lib.X509_NAME_get_entry(x509_name, x) + attribute = _decode_x509_name_entry(backend, entry) + set_id = backend._lib.X509_NAME_ENTRY_set(entry) + if set_id != prev_set_id: + attributes.append({attribute}) + else: + # is in the same RDN a previous entry + attributes[-1].add(attribute) + prev_set_id = set_id + + return x509.Name(x509.RelativeDistinguishedName(rdn) for rdn in attributes) + + +def _decode_general_names(backend, gns): + num = backend._lib.sk_GENERAL_NAME_num(gns) + names = [] + for i in range(num): + gn = backend._lib.sk_GENERAL_NAME_value(gns, i) + backend.openssl_assert(gn != backend._ffi.NULL) + names.append(_decode_general_name(backend, gn)) + + return names + + +def _decode_general_name(backend, gn): + if gn.type == backend._lib.GEN_DNS: + # Convert to bytes and then decode to utf8. We don't use + # asn1_string_to_utf8 here because it doesn't properly convert + # utf8 from ia5strings. + data = _asn1_string_to_bytes(backend, gn.d.dNSName).decode("utf8") + # We don't use the constructor for DNSName so we can bypass validation + # This allows us to create DNSName objects that have unicode chars + # when a certificate (against the RFC) contains them. + return x509.DNSName._init_without_validation(data) + elif gn.type == backend._lib.GEN_URI: + # Convert to bytes and then decode to utf8. We don't use + # asn1_string_to_utf8 here because it doesn't properly convert + # utf8 from ia5strings. + data = _asn1_string_to_bytes( + backend, gn.d.uniformResourceIdentifier + ).decode("utf8") + # We don't use the constructor for URI so we can bypass validation + # This allows us to create URI objects that have unicode chars + # when a certificate (against the RFC) contains them. + return x509.UniformResourceIdentifier._init_without_validation(data) + elif gn.type == backend._lib.GEN_RID: + oid = _obj2txt(backend, gn.d.registeredID) + return x509.RegisteredID(x509.ObjectIdentifier(oid)) + elif gn.type == backend._lib.GEN_IPADD: + data = _asn1_string_to_bytes(backend, gn.d.iPAddress) + data_len = len(data) + if data_len == 8 or data_len == 32: + # This is an IPv4 or IPv6 Network and not a single IP. This + # type of data appears in Name Constraints. Unfortunately, + # ipaddress doesn't support packed bytes + netmask. Additionally, + # IPv6Network can only handle CIDR rather than the full 16 byte + # netmask. To handle this we convert the netmask to integer, then + # find the first 0 bit, which will be the prefix. If another 1 + # bit is present after that the netmask is invalid. + base = ipaddress.ip_address(data[: data_len // 2]) + netmask = ipaddress.ip_address(data[data_len // 2 :]) + bits = bin(int(netmask))[2:] + prefix = bits.find("0") + # If no 0 bits are found it is a /32 or /128 + if prefix == -1: + prefix = len(bits) + + if "1" in bits[prefix:]: + raise ValueError("Invalid netmask") + + ip = ipaddress.ip_network(base.exploded + u"/{}".format(prefix)) + else: + ip = ipaddress.ip_address(data) + + return x509.IPAddress(ip) + elif gn.type == backend._lib.GEN_DIRNAME: + return x509.DirectoryName( + _decode_x509_name(backend, gn.d.directoryName) + ) + elif gn.type == backend._lib.GEN_EMAIL: + # Convert to bytes and then decode to utf8. We don't use + # asn1_string_to_utf8 here because it doesn't properly convert + # utf8 from ia5strings. + data = _asn1_string_to_bytes(backend, gn.d.rfc822Name).decode("utf8") + # We don't use the constructor for RFC822Name so we can bypass + # validation. This allows us to create RFC822Name objects that have + # unicode chars when a certificate (against the RFC) contains them. + return x509.RFC822Name._init_without_validation(data) + elif gn.type == backend._lib.GEN_OTHERNAME: + type_id = _obj2txt(backend, gn.d.otherName.type_id) + value = _asn1_to_der(backend, gn.d.otherName.value) + return x509.OtherName(x509.ObjectIdentifier(type_id), value) + else: + # x400Address or ediPartyName + raise x509.UnsupportedGeneralNameType( + "{} is not a supported type".format( + x509._GENERAL_NAMES.get(gn.type, gn.type) + ), + gn.type, + ) + + +def _decode_ocsp_no_check(backend, ext): + return x509.OCSPNoCheck() + + +def _decode_crl_number(backend, ext): + asn1_int = backend._ffi.cast("ASN1_INTEGER *", ext) + asn1_int = backend._ffi.gc(asn1_int, backend._lib.ASN1_INTEGER_free) + return x509.CRLNumber(_asn1_integer_to_int(backend, asn1_int)) + + +def _decode_delta_crl_indicator(backend, ext): + asn1_int = backend._ffi.cast("ASN1_INTEGER *", ext) + asn1_int = backend._ffi.gc(asn1_int, backend._lib.ASN1_INTEGER_free) + return x509.DeltaCRLIndicator(_asn1_integer_to_int(backend, asn1_int)) + + +class _X509ExtensionParser(object): + def __init__(self, backend, ext_count, get_ext, handlers): + self.ext_count = ext_count + self.get_ext = get_ext + self.handlers = handlers + self._backend = backend + + def parse(self, x509_obj): + extensions = [] + seen_oids = set() + for i in range(self.ext_count(x509_obj)): + ext = self.get_ext(x509_obj, i) + self._backend.openssl_assert(ext != self._backend._ffi.NULL) + crit = self._backend._lib.X509_EXTENSION_get_critical(ext) + critical = crit == 1 + oid = x509.ObjectIdentifier( + _obj2txt( + self._backend, + self._backend._lib.X509_EXTENSION_get_object(ext), + ) + ) + if oid in seen_oids: + raise x509.DuplicateExtension( + "Duplicate {} extension found".format(oid), oid + ) + + # These OIDs are only supported in OpenSSL 1.1.0+ but we want + # to support them in all versions of OpenSSL so we decode them + # ourselves. + if oid == ExtensionOID.TLS_FEATURE: + # The extension contents are a SEQUENCE OF INTEGERs. + data = self._backend._lib.X509_EXTENSION_get_data(ext) + data_bytes = _asn1_string_to_bytes(self._backend, data) + features = DERReader(data_bytes).read_single_element(SEQUENCE) + parsed = [] + while not features.is_empty(): + parsed.append(features.read_element(INTEGER).as_integer()) + # Map the features to their enum value. + value = x509.TLSFeature( + [_TLS_FEATURE_TYPE_TO_ENUM[x] for x in parsed] + ) + extensions.append(x509.Extension(oid, critical, value)) + seen_oids.add(oid) + continue + elif oid == ExtensionOID.PRECERT_POISON: + data = self._backend._lib.X509_EXTENSION_get_data(ext) + # The contents of the extension must be an ASN.1 NULL. + reader = DERReader(_asn1_string_to_bytes(self._backend, data)) + reader.read_single_element(NULL).check_empty() + extensions.append( + x509.Extension(oid, critical, x509.PrecertPoison()) + ) + seen_oids.add(oid) + continue + + try: + handler = self.handlers[oid] + except KeyError: + # Dump the DER payload into an UnrecognizedExtension object + data = self._backend._lib.X509_EXTENSION_get_data(ext) + self._backend.openssl_assert(data != self._backend._ffi.NULL) + der = self._backend._ffi.buffer(data.data, data.length)[:] + unrecognized = x509.UnrecognizedExtension(oid, der) + extensions.append(x509.Extension(oid, critical, unrecognized)) + else: + ext_data = self._backend._lib.X509V3_EXT_d2i(ext) + if ext_data == self._backend._ffi.NULL: + self._backend._consume_errors() + raise ValueError( + "The {} extension is invalid and can't be " + "parsed".format(oid) + ) + + value = handler(self._backend, ext_data) + extensions.append(x509.Extension(oid, critical, value)) + + seen_oids.add(oid) + + return x509.Extensions(extensions) + + +def _decode_certificate_policies(backend, cp): + cp = backend._ffi.cast("Cryptography_STACK_OF_POLICYINFO *", cp) + cp = backend._ffi.gc(cp, backend._lib.CERTIFICATEPOLICIES_free) + + num = backend._lib.sk_POLICYINFO_num(cp) + certificate_policies = [] + for i in range(num): + qualifiers = None + pi = backend._lib.sk_POLICYINFO_value(cp, i) + oid = x509.ObjectIdentifier(_obj2txt(backend, pi.policyid)) + if pi.qualifiers != backend._ffi.NULL: + qnum = backend._lib.sk_POLICYQUALINFO_num(pi.qualifiers) + qualifiers = [] + for j in range(qnum): + pqi = backend._lib.sk_POLICYQUALINFO_value(pi.qualifiers, j) + pqualid = x509.ObjectIdentifier(_obj2txt(backend, pqi.pqualid)) + if pqualid == CertificatePoliciesOID.CPS_QUALIFIER: + cpsuri = backend._ffi.buffer( + pqi.d.cpsuri.data, pqi.d.cpsuri.length + )[:].decode("ascii") + qualifiers.append(cpsuri) + else: + assert pqualid == CertificatePoliciesOID.CPS_USER_NOTICE + user_notice = _decode_user_notice( + backend, pqi.d.usernotice + ) + qualifiers.append(user_notice) + + certificate_policies.append(x509.PolicyInformation(oid, qualifiers)) + + return x509.CertificatePolicies(certificate_policies) + + +def _decode_user_notice(backend, un): + explicit_text = None + notice_reference = None + + if un.exptext != backend._ffi.NULL: + explicit_text = _asn1_string_to_utf8(backend, un.exptext) + + if un.noticeref != backend._ffi.NULL: + organization = _asn1_string_to_utf8(backend, un.noticeref.organization) + + num = backend._lib.sk_ASN1_INTEGER_num(un.noticeref.noticenos) + notice_numbers = [] + for i in range(num): + asn1_int = backend._lib.sk_ASN1_INTEGER_value( + un.noticeref.noticenos, i + ) + notice_num = _asn1_integer_to_int(backend, asn1_int) + notice_numbers.append(notice_num) + + notice_reference = x509.NoticeReference(organization, notice_numbers) + + return x509.UserNotice(notice_reference, explicit_text) + + +def _decode_basic_constraints(backend, bc_st): + basic_constraints = backend._ffi.cast("BASIC_CONSTRAINTS *", bc_st) + basic_constraints = backend._ffi.gc( + basic_constraints, backend._lib.BASIC_CONSTRAINTS_free + ) + # The byte representation of an ASN.1 boolean true is \xff. OpenSSL + # chooses to just map this to its ordinal value, so true is 255 and + # false is 0. + ca = basic_constraints.ca == 255 + path_length = _asn1_integer_to_int_or_none( + backend, basic_constraints.pathlen + ) + + return x509.BasicConstraints(ca, path_length) + + +def _decode_subject_key_identifier(backend, asn1_string): + asn1_string = backend._ffi.cast("ASN1_OCTET_STRING *", asn1_string) + asn1_string = backend._ffi.gc( + asn1_string, backend._lib.ASN1_OCTET_STRING_free + ) + return x509.SubjectKeyIdentifier( + backend._ffi.buffer(asn1_string.data, asn1_string.length)[:] + ) + + +def _decode_authority_key_identifier(backend, akid): + akid = backend._ffi.cast("AUTHORITY_KEYID *", akid) + akid = backend._ffi.gc(akid, backend._lib.AUTHORITY_KEYID_free) + key_identifier = None + authority_cert_issuer = None + + if akid.keyid != backend._ffi.NULL: + key_identifier = backend._ffi.buffer( + akid.keyid.data, akid.keyid.length + )[:] + + if akid.issuer != backend._ffi.NULL: + authority_cert_issuer = _decode_general_names(backend, akid.issuer) + + authority_cert_serial_number = _asn1_integer_to_int_or_none( + backend, akid.serial + ) + + return x509.AuthorityKeyIdentifier( + key_identifier, authority_cert_issuer, authority_cert_serial_number + ) + + +def _decode_information_access(backend, ia): + ia = backend._ffi.cast("Cryptography_STACK_OF_ACCESS_DESCRIPTION *", ia) + ia = backend._ffi.gc( + ia, + lambda x: backend._lib.sk_ACCESS_DESCRIPTION_pop_free( + x, + backend._ffi.addressof( + backend._lib._original_lib, "ACCESS_DESCRIPTION_free" + ), + ), + ) + num = backend._lib.sk_ACCESS_DESCRIPTION_num(ia) + access_descriptions = [] + for i in range(num): + ad = backend._lib.sk_ACCESS_DESCRIPTION_value(ia, i) + backend.openssl_assert(ad.method != backend._ffi.NULL) + oid = x509.ObjectIdentifier(_obj2txt(backend, ad.method)) + backend.openssl_assert(ad.location != backend._ffi.NULL) + gn = _decode_general_name(backend, ad.location) + access_descriptions.append(x509.AccessDescription(oid, gn)) + + return access_descriptions + + +def _decode_authority_information_access(backend, aia): + access_descriptions = _decode_information_access(backend, aia) + return x509.AuthorityInformationAccess(access_descriptions) + + +def _decode_subject_information_access(backend, aia): + access_descriptions = _decode_information_access(backend, aia) + return x509.SubjectInformationAccess(access_descriptions) + + +def _decode_key_usage(backend, bit_string): + bit_string = backend._ffi.cast("ASN1_BIT_STRING *", bit_string) + bit_string = backend._ffi.gc(bit_string, backend._lib.ASN1_BIT_STRING_free) + get_bit = backend._lib.ASN1_BIT_STRING_get_bit + digital_signature = get_bit(bit_string, 0) == 1 + content_commitment = get_bit(bit_string, 1) == 1 + key_encipherment = get_bit(bit_string, 2) == 1 + data_encipherment = get_bit(bit_string, 3) == 1 + key_agreement = get_bit(bit_string, 4) == 1 + key_cert_sign = get_bit(bit_string, 5) == 1 + crl_sign = get_bit(bit_string, 6) == 1 + encipher_only = get_bit(bit_string, 7) == 1 + decipher_only = get_bit(bit_string, 8) == 1 + return x509.KeyUsage( + digital_signature, + content_commitment, + key_encipherment, + data_encipherment, + key_agreement, + key_cert_sign, + crl_sign, + encipher_only, + decipher_only, + ) + + +def _decode_general_names_extension(backend, gns): + gns = backend._ffi.cast("GENERAL_NAMES *", gns) + gns = backend._ffi.gc(gns, backend._lib.GENERAL_NAMES_free) + general_names = _decode_general_names(backend, gns) + return general_names + + +def _decode_subject_alt_name(backend, ext): + return x509.SubjectAlternativeName( + _decode_general_names_extension(backend, ext) + ) + + +def _decode_issuer_alt_name(backend, ext): + return x509.IssuerAlternativeName( + _decode_general_names_extension(backend, ext) + ) + + +def _decode_name_constraints(backend, nc): + nc = backend._ffi.cast("NAME_CONSTRAINTS *", nc) + nc = backend._ffi.gc(nc, backend._lib.NAME_CONSTRAINTS_free) + permitted = _decode_general_subtrees(backend, nc.permittedSubtrees) + excluded = _decode_general_subtrees(backend, nc.excludedSubtrees) + return x509.NameConstraints( + permitted_subtrees=permitted, excluded_subtrees=excluded + ) + + +def _decode_general_subtrees(backend, stack_subtrees): + if stack_subtrees == backend._ffi.NULL: + return None + + num = backend._lib.sk_GENERAL_SUBTREE_num(stack_subtrees) + subtrees = [] + + for i in range(num): + obj = backend._lib.sk_GENERAL_SUBTREE_value(stack_subtrees, i) + backend.openssl_assert(obj != backend._ffi.NULL) + name = _decode_general_name(backend, obj.base) + subtrees.append(name) + + return subtrees + + +def _decode_issuing_dist_point(backend, idp): + idp = backend._ffi.cast("ISSUING_DIST_POINT *", idp) + idp = backend._ffi.gc(idp, backend._lib.ISSUING_DIST_POINT_free) + if idp.distpoint != backend._ffi.NULL: + full_name, relative_name = _decode_distpoint(backend, idp.distpoint) + else: + full_name = None + relative_name = None + + only_user = idp.onlyuser == 255 + only_ca = idp.onlyCA == 255 + indirect_crl = idp.indirectCRL == 255 + only_attr = idp.onlyattr == 255 + if idp.onlysomereasons != backend._ffi.NULL: + only_some_reasons = _decode_reasons(backend, idp.onlysomereasons) + else: + only_some_reasons = None + + return x509.IssuingDistributionPoint( + full_name, + relative_name, + only_user, + only_ca, + only_some_reasons, + indirect_crl, + only_attr, + ) + + +def _decode_policy_constraints(backend, pc): + pc = backend._ffi.cast("POLICY_CONSTRAINTS *", pc) + pc = backend._ffi.gc(pc, backend._lib.POLICY_CONSTRAINTS_free) + + require_explicit_policy = _asn1_integer_to_int_or_none( + backend, pc.requireExplicitPolicy + ) + inhibit_policy_mapping = _asn1_integer_to_int_or_none( + backend, pc.inhibitPolicyMapping + ) + + return x509.PolicyConstraints( + require_explicit_policy, inhibit_policy_mapping + ) + + +def _decode_extended_key_usage(backend, sk): + sk = backend._ffi.cast("Cryptography_STACK_OF_ASN1_OBJECT *", sk) + sk = backend._ffi.gc(sk, backend._lib.sk_ASN1_OBJECT_free) + num = backend._lib.sk_ASN1_OBJECT_num(sk) + ekus = [] + + for i in range(num): + obj = backend._lib.sk_ASN1_OBJECT_value(sk, i) + backend.openssl_assert(obj != backend._ffi.NULL) + oid = x509.ObjectIdentifier(_obj2txt(backend, obj)) + ekus.append(oid) + + return x509.ExtendedKeyUsage(ekus) + + +_DISTPOINT_TYPE_FULLNAME = 0 +_DISTPOINT_TYPE_RELATIVENAME = 1 + + +def _decode_dist_points(backend, cdps): + cdps = backend._ffi.cast("Cryptography_STACK_OF_DIST_POINT *", cdps) + cdps = backend._ffi.gc(cdps, backend._lib.CRL_DIST_POINTS_free) + + num = backend._lib.sk_DIST_POINT_num(cdps) + dist_points = [] + for i in range(num): + full_name = None + relative_name = None + crl_issuer = None + reasons = None + cdp = backend._lib.sk_DIST_POINT_value(cdps, i) + if cdp.reasons != backend._ffi.NULL: + reasons = _decode_reasons(backend, cdp.reasons) + + if cdp.CRLissuer != backend._ffi.NULL: + crl_issuer = _decode_general_names(backend, cdp.CRLissuer) + + # Certificates may have a crl_issuer/reasons and no distribution + # point so make sure it's not null. + if cdp.distpoint != backend._ffi.NULL: + full_name, relative_name = _decode_distpoint( + backend, cdp.distpoint + ) + + dist_points.append( + x509.DistributionPoint( + full_name, relative_name, reasons, crl_issuer + ) + ) + + return dist_points + + +# ReasonFlags ::= BIT STRING { +# unused (0), +# keyCompromise (1), +# cACompromise (2), +# affiliationChanged (3), +# superseded (4), +# cessationOfOperation (5), +# certificateHold (6), +# privilegeWithdrawn (7), +# aACompromise (8) } +_REASON_BIT_MAPPING = { + 1: x509.ReasonFlags.key_compromise, + 2: x509.ReasonFlags.ca_compromise, + 3: x509.ReasonFlags.affiliation_changed, + 4: x509.ReasonFlags.superseded, + 5: x509.ReasonFlags.cessation_of_operation, + 6: x509.ReasonFlags.certificate_hold, + 7: x509.ReasonFlags.privilege_withdrawn, + 8: x509.ReasonFlags.aa_compromise, +} + + +def _decode_reasons(backend, reasons): + # We will check each bit from RFC 5280 + enum_reasons = [] + for bit_position, reason in six.iteritems(_REASON_BIT_MAPPING): + if backend._lib.ASN1_BIT_STRING_get_bit(reasons, bit_position): + enum_reasons.append(reason) + + return frozenset(enum_reasons) + + +def _decode_distpoint(backend, distpoint): + if distpoint.type == _DISTPOINT_TYPE_FULLNAME: + full_name = _decode_general_names(backend, distpoint.name.fullname) + return full_name, None + + # OpenSSL code doesn't test for a specific type for + # relativename, everything that isn't fullname is considered + # relativename. Per RFC 5280: + # + # DistributionPointName ::= CHOICE { + # fullName [0] GeneralNames, + # nameRelativeToCRLIssuer [1] RelativeDistinguishedName } + rns = distpoint.name.relativename + rnum = backend._lib.sk_X509_NAME_ENTRY_num(rns) + attributes = set() + for i in range(rnum): + rn = backend._lib.sk_X509_NAME_ENTRY_value(rns, i) + backend.openssl_assert(rn != backend._ffi.NULL) + attributes.add(_decode_x509_name_entry(backend, rn)) + + relative_name = x509.RelativeDistinguishedName(attributes) + + return None, relative_name + + +def _decode_crl_distribution_points(backend, cdps): + dist_points = _decode_dist_points(backend, cdps) + return x509.CRLDistributionPoints(dist_points) + + +def _decode_freshest_crl(backend, cdps): + dist_points = _decode_dist_points(backend, cdps) + return x509.FreshestCRL(dist_points) + + +def _decode_inhibit_any_policy(backend, asn1_int): + asn1_int = backend._ffi.cast("ASN1_INTEGER *", asn1_int) + asn1_int = backend._ffi.gc(asn1_int, backend._lib.ASN1_INTEGER_free) + skip_certs = _asn1_integer_to_int(backend, asn1_int) + return x509.InhibitAnyPolicy(skip_certs) + + +def _decode_scts(backend, asn1_scts): + from cryptography.hazmat.backends.openssl.x509 import ( + _SignedCertificateTimestamp, + ) + + asn1_scts = backend._ffi.cast("Cryptography_STACK_OF_SCT *", asn1_scts) + asn1_scts = backend._ffi.gc(asn1_scts, backend._lib.SCT_LIST_free) + + scts = [] + for i in range(backend._lib.sk_SCT_num(asn1_scts)): + sct = backend._lib.sk_SCT_value(asn1_scts, i) + + scts.append(_SignedCertificateTimestamp(backend, asn1_scts, sct)) + return scts + + +def _decode_precert_signed_certificate_timestamps(backend, asn1_scts): + return x509.PrecertificateSignedCertificateTimestamps( + _decode_scts(backend, asn1_scts) + ) + + +def _decode_signed_certificate_timestamps(backend, asn1_scts): + return x509.SignedCertificateTimestamps(_decode_scts(backend, asn1_scts)) + + +# CRLReason ::= ENUMERATED { +# unspecified (0), +# keyCompromise (1), +# cACompromise (2), +# affiliationChanged (3), +# superseded (4), +# cessationOfOperation (5), +# certificateHold (6), +# -- value 7 is not used +# removeFromCRL (8), +# privilegeWithdrawn (9), +# aACompromise (10) } +_CRL_ENTRY_REASON_CODE_TO_ENUM = { + 0: x509.ReasonFlags.unspecified, + 1: x509.ReasonFlags.key_compromise, + 2: x509.ReasonFlags.ca_compromise, + 3: x509.ReasonFlags.affiliation_changed, + 4: x509.ReasonFlags.superseded, + 5: x509.ReasonFlags.cessation_of_operation, + 6: x509.ReasonFlags.certificate_hold, + 8: x509.ReasonFlags.remove_from_crl, + 9: x509.ReasonFlags.privilege_withdrawn, + 10: x509.ReasonFlags.aa_compromise, +} + + +_CRL_ENTRY_REASON_ENUM_TO_CODE = { + x509.ReasonFlags.unspecified: 0, + x509.ReasonFlags.key_compromise: 1, + x509.ReasonFlags.ca_compromise: 2, + x509.ReasonFlags.affiliation_changed: 3, + x509.ReasonFlags.superseded: 4, + x509.ReasonFlags.cessation_of_operation: 5, + x509.ReasonFlags.certificate_hold: 6, + x509.ReasonFlags.remove_from_crl: 8, + x509.ReasonFlags.privilege_withdrawn: 9, + x509.ReasonFlags.aa_compromise: 10, +} + + +def _decode_crl_reason(backend, enum): + enum = backend._ffi.cast("ASN1_ENUMERATED *", enum) + enum = backend._ffi.gc(enum, backend._lib.ASN1_ENUMERATED_free) + code = backend._lib.ASN1_ENUMERATED_get(enum) + + try: + return x509.CRLReason(_CRL_ENTRY_REASON_CODE_TO_ENUM[code]) + except KeyError: + raise ValueError("Unsupported reason code: {}".format(code)) + + +def _decode_invalidity_date(backend, inv_date): + generalized_time = backend._ffi.cast("ASN1_GENERALIZEDTIME *", inv_date) + generalized_time = backend._ffi.gc( + generalized_time, backend._lib.ASN1_GENERALIZEDTIME_free + ) + return x509.InvalidityDate( + _parse_asn1_generalized_time(backend, generalized_time) + ) + + +def _decode_cert_issuer(backend, gns): + gns = backend._ffi.cast("GENERAL_NAMES *", gns) + gns = backend._ffi.gc(gns, backend._lib.GENERAL_NAMES_free) + general_names = _decode_general_names(backend, gns) + return x509.CertificateIssuer(general_names) + + +def _asn1_to_der(backend, asn1_type): + buf = backend._ffi.new("unsigned char **") + res = backend._lib.i2d_ASN1_TYPE(asn1_type, buf) + backend.openssl_assert(res >= 0) + backend.openssl_assert(buf[0] != backend._ffi.NULL) + buf = backend._ffi.gc( + buf, lambda buffer: backend._lib.OPENSSL_free(buffer[0]) + ) + return backend._ffi.buffer(buf[0], res)[:] + + +def _asn1_integer_to_int(backend, asn1_int): + bn = backend._lib.ASN1_INTEGER_to_BN(asn1_int, backend._ffi.NULL) + backend.openssl_assert(bn != backend._ffi.NULL) + bn = backend._ffi.gc(bn, backend._lib.BN_free) + return backend._bn_to_int(bn) + + +def _asn1_integer_to_int_or_none(backend, asn1_int): + if asn1_int == backend._ffi.NULL: + return None + else: + return _asn1_integer_to_int(backend, asn1_int) + + +def _asn1_string_to_bytes(backend, asn1_string): + return backend._ffi.buffer(asn1_string.data, asn1_string.length)[:] + + +def _asn1_string_to_ascii(backend, asn1_string): + return _asn1_string_to_bytes(backend, asn1_string).decode("ascii") + + +def _asn1_string_to_utf8(backend, asn1_string): + buf = backend._ffi.new("unsigned char **") + res = backend._lib.ASN1_STRING_to_UTF8(buf, asn1_string) + if res == -1: + raise ValueError( + "Unsupported ASN1 string type. Type: {}".format(asn1_string.type) + ) + + backend.openssl_assert(buf[0] != backend._ffi.NULL) + buf = backend._ffi.gc( + buf, lambda buffer: backend._lib.OPENSSL_free(buffer[0]) + ) + return backend._ffi.buffer(buf[0], res)[:].decode("utf8") + + +def _parse_asn1_time(backend, asn1_time): + backend.openssl_assert(asn1_time != backend._ffi.NULL) + generalized_time = backend._lib.ASN1_TIME_to_generalizedtime( + asn1_time, backend._ffi.NULL + ) + if generalized_time == backend._ffi.NULL: + raise ValueError( + "Couldn't parse ASN.1 time as generalizedtime {!r}".format( + _asn1_string_to_bytes(backend, asn1_time) + ) + ) + + generalized_time = backend._ffi.gc( + generalized_time, backend._lib.ASN1_GENERALIZEDTIME_free + ) + return _parse_asn1_generalized_time(backend, generalized_time) + + +def _parse_asn1_generalized_time(backend, generalized_time): + time = _asn1_string_to_ascii( + backend, backend._ffi.cast("ASN1_STRING *", generalized_time) + ) + return datetime.datetime.strptime(time, "%Y%m%d%H%M%SZ") + + +def _decode_nonce(backend, nonce): + nonce = backend._ffi.cast("ASN1_OCTET_STRING *", nonce) + nonce = backend._ffi.gc(nonce, backend._lib.ASN1_OCTET_STRING_free) + return x509.OCSPNonce(_asn1_string_to_bytes(backend, nonce)) + + +_EXTENSION_HANDLERS_BASE = { + ExtensionOID.BASIC_CONSTRAINTS: _decode_basic_constraints, + ExtensionOID.SUBJECT_KEY_IDENTIFIER: _decode_subject_key_identifier, + ExtensionOID.KEY_USAGE: _decode_key_usage, + ExtensionOID.SUBJECT_ALTERNATIVE_NAME: _decode_subject_alt_name, + ExtensionOID.EXTENDED_KEY_USAGE: _decode_extended_key_usage, + ExtensionOID.AUTHORITY_KEY_IDENTIFIER: _decode_authority_key_identifier, + ExtensionOID.AUTHORITY_INFORMATION_ACCESS: ( + _decode_authority_information_access + ), + ExtensionOID.SUBJECT_INFORMATION_ACCESS: ( + _decode_subject_information_access + ), + ExtensionOID.CERTIFICATE_POLICIES: _decode_certificate_policies, + ExtensionOID.CRL_DISTRIBUTION_POINTS: _decode_crl_distribution_points, + ExtensionOID.FRESHEST_CRL: _decode_freshest_crl, + ExtensionOID.OCSP_NO_CHECK: _decode_ocsp_no_check, + ExtensionOID.INHIBIT_ANY_POLICY: _decode_inhibit_any_policy, + ExtensionOID.ISSUER_ALTERNATIVE_NAME: _decode_issuer_alt_name, + ExtensionOID.NAME_CONSTRAINTS: _decode_name_constraints, + ExtensionOID.POLICY_CONSTRAINTS: _decode_policy_constraints, +} +_EXTENSION_HANDLERS_SCT = { + ExtensionOID.PRECERT_SIGNED_CERTIFICATE_TIMESTAMPS: ( + _decode_precert_signed_certificate_timestamps + ) +} + +_REVOKED_EXTENSION_HANDLERS = { + CRLEntryExtensionOID.CRL_REASON: _decode_crl_reason, + CRLEntryExtensionOID.INVALIDITY_DATE: _decode_invalidity_date, + CRLEntryExtensionOID.CERTIFICATE_ISSUER: _decode_cert_issuer, +} + +_CRL_EXTENSION_HANDLERS = { + ExtensionOID.CRL_NUMBER: _decode_crl_number, + ExtensionOID.DELTA_CRL_INDICATOR: _decode_delta_crl_indicator, + ExtensionOID.AUTHORITY_KEY_IDENTIFIER: _decode_authority_key_identifier, + ExtensionOID.ISSUER_ALTERNATIVE_NAME: _decode_issuer_alt_name, + ExtensionOID.AUTHORITY_INFORMATION_ACCESS: ( + _decode_authority_information_access + ), + ExtensionOID.ISSUING_DISTRIBUTION_POINT: _decode_issuing_dist_point, + ExtensionOID.FRESHEST_CRL: _decode_freshest_crl, +} + +_OCSP_REQ_EXTENSION_HANDLERS = { + OCSPExtensionOID.NONCE: _decode_nonce, +} + +_OCSP_BASICRESP_EXTENSION_HANDLERS = { + OCSPExtensionOID.NONCE: _decode_nonce, +} + +_OCSP_SINGLERESP_EXTENSION_HANDLERS_SCT = { + ExtensionOID.SIGNED_CERTIFICATE_TIMESTAMPS: ( + _decode_signed_certificate_timestamps + ) +} diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/openssl/dh.py b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/openssl/dh.py new file mode 100644 index 0000000..2862676 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/openssl/dh.py @@ -0,0 +1,271 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function + +from cryptography import utils +from cryptography.exceptions import UnsupportedAlgorithm, _Reasons +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.asymmetric import dh + + +def _dh_params_dup(dh_cdata, backend): + lib = backend._lib + ffi = backend._ffi + + param_cdata = lib.DHparams_dup(dh_cdata) + backend.openssl_assert(param_cdata != ffi.NULL) + param_cdata = ffi.gc(param_cdata, lib.DH_free) + if lib.CRYPTOGRAPHY_IS_LIBRESSL: + # In libressl DHparams_dup don't copy q + q = ffi.new("BIGNUM **") + lib.DH_get0_pqg(dh_cdata, ffi.NULL, q, ffi.NULL) + q_dup = lib.BN_dup(q[0]) + res = lib.DH_set0_pqg(param_cdata, ffi.NULL, q_dup, ffi.NULL) + backend.openssl_assert(res == 1) + + return param_cdata + + +def _dh_cdata_to_parameters(dh_cdata, backend): + param_cdata = _dh_params_dup(dh_cdata, backend) + return _DHParameters(backend, param_cdata) + + +@utils.register_interface(dh.DHParametersWithSerialization) +class _DHParameters(object): + def __init__(self, backend, dh_cdata): + self._backend = backend + self._dh_cdata = dh_cdata + + def parameter_numbers(self): + p = self._backend._ffi.new("BIGNUM **") + g = self._backend._ffi.new("BIGNUM **") + q = self._backend._ffi.new("BIGNUM **") + self._backend._lib.DH_get0_pqg(self._dh_cdata, p, q, g) + self._backend.openssl_assert(p[0] != self._backend._ffi.NULL) + self._backend.openssl_assert(g[0] != self._backend._ffi.NULL) + if q[0] == self._backend._ffi.NULL: + q_val = None + else: + q_val = self._backend._bn_to_int(q[0]) + return dh.DHParameterNumbers( + p=self._backend._bn_to_int(p[0]), + g=self._backend._bn_to_int(g[0]), + q=q_val, + ) + + def generate_private_key(self): + return self._backend.generate_dh_private_key(self) + + def parameter_bytes(self, encoding, format): + if format is not serialization.ParameterFormat.PKCS3: + raise ValueError("Only PKCS3 serialization is supported") + if not self._backend._lib.Cryptography_HAS_EVP_PKEY_DHX: + q = self._backend._ffi.new("BIGNUM **") + self._backend._lib.DH_get0_pqg( + self._dh_cdata, + self._backend._ffi.NULL, + q, + self._backend._ffi.NULL, + ) + if q[0] != self._backend._ffi.NULL: + raise UnsupportedAlgorithm( + "DH X9.42 serialization is not supported", + _Reasons.UNSUPPORTED_SERIALIZATION, + ) + + return self._backend._parameter_bytes(encoding, format, self._dh_cdata) + + +def _get_dh_num_bits(backend, dh_cdata): + p = backend._ffi.new("BIGNUM **") + backend._lib.DH_get0_pqg(dh_cdata, p, backend._ffi.NULL, backend._ffi.NULL) + backend.openssl_assert(p[0] != backend._ffi.NULL) + return backend._lib.BN_num_bits(p[0]) + + +@utils.register_interface(dh.DHPrivateKeyWithSerialization) +class _DHPrivateKey(object): + def __init__(self, backend, dh_cdata, evp_pkey): + self._backend = backend + self._dh_cdata = dh_cdata + self._evp_pkey = evp_pkey + self._key_size_bytes = self._backend._lib.DH_size(dh_cdata) + + @property + def key_size(self): + return _get_dh_num_bits(self._backend, self._dh_cdata) + + def private_numbers(self): + p = self._backend._ffi.new("BIGNUM **") + g = self._backend._ffi.new("BIGNUM **") + q = self._backend._ffi.new("BIGNUM **") + self._backend._lib.DH_get0_pqg(self._dh_cdata, p, q, g) + self._backend.openssl_assert(p[0] != self._backend._ffi.NULL) + self._backend.openssl_assert(g[0] != self._backend._ffi.NULL) + if q[0] == self._backend._ffi.NULL: + q_val = None + else: + q_val = self._backend._bn_to_int(q[0]) + pub_key = self._backend._ffi.new("BIGNUM **") + priv_key = self._backend._ffi.new("BIGNUM **") + self._backend._lib.DH_get0_key(self._dh_cdata, pub_key, priv_key) + self._backend.openssl_assert(pub_key[0] != self._backend._ffi.NULL) + self._backend.openssl_assert(priv_key[0] != self._backend._ffi.NULL) + return dh.DHPrivateNumbers( + public_numbers=dh.DHPublicNumbers( + parameter_numbers=dh.DHParameterNumbers( + p=self._backend._bn_to_int(p[0]), + g=self._backend._bn_to_int(g[0]), + q=q_val, + ), + y=self._backend._bn_to_int(pub_key[0]), + ), + x=self._backend._bn_to_int(priv_key[0]), + ) + + def exchange(self, peer_public_key): + + buf = self._backend._ffi.new("unsigned char[]", self._key_size_bytes) + pub_key = self._backend._ffi.new("BIGNUM **") + self._backend._lib.DH_get0_key( + peer_public_key._dh_cdata, pub_key, self._backend._ffi.NULL + ) + self._backend.openssl_assert(pub_key[0] != self._backend._ffi.NULL) + res = self._backend._lib.DH_compute_key( + buf, pub_key[0], self._dh_cdata + ) + + if res == -1: + errors_with_text = self._backend._consume_errors_with_text() + raise ValueError( + "Error computing shared key. Public key is likely invalid " + "for this exchange.", + errors_with_text, + ) + else: + self._backend.openssl_assert(res >= 1) + + key = self._backend._ffi.buffer(buf)[:res] + pad = self._key_size_bytes - len(key) + + if pad > 0: + key = (b"\x00" * pad) + key + + return key + + def public_key(self): + dh_cdata = _dh_params_dup(self._dh_cdata, self._backend) + pub_key = self._backend._ffi.new("BIGNUM **") + self._backend._lib.DH_get0_key( + self._dh_cdata, pub_key, self._backend._ffi.NULL + ) + self._backend.openssl_assert(pub_key[0] != self._backend._ffi.NULL) + pub_key_dup = self._backend._lib.BN_dup(pub_key[0]) + self._backend.openssl_assert(pub_key_dup != self._backend._ffi.NULL) + + res = self._backend._lib.DH_set0_key( + dh_cdata, pub_key_dup, self._backend._ffi.NULL + ) + self._backend.openssl_assert(res == 1) + evp_pkey = self._backend._dh_cdata_to_evp_pkey(dh_cdata) + return _DHPublicKey(self._backend, dh_cdata, evp_pkey) + + def parameters(self): + return _dh_cdata_to_parameters(self._dh_cdata, self._backend) + + def private_bytes(self, encoding, format, encryption_algorithm): + if format is not serialization.PrivateFormat.PKCS8: + raise ValueError( + "DH private keys support only PKCS8 serialization" + ) + if not self._backend._lib.Cryptography_HAS_EVP_PKEY_DHX: + q = self._backend._ffi.new("BIGNUM **") + self._backend._lib.DH_get0_pqg( + self._dh_cdata, + self._backend._ffi.NULL, + q, + self._backend._ffi.NULL, + ) + if q[0] != self._backend._ffi.NULL: + raise UnsupportedAlgorithm( + "DH X9.42 serialization is not supported", + _Reasons.UNSUPPORTED_SERIALIZATION, + ) + + return self._backend._private_key_bytes( + encoding, + format, + encryption_algorithm, + self, + self._evp_pkey, + self._dh_cdata, + ) + + +@utils.register_interface(dh.DHPublicKeyWithSerialization) +class _DHPublicKey(object): + def __init__(self, backend, dh_cdata, evp_pkey): + self._backend = backend + self._dh_cdata = dh_cdata + self._evp_pkey = evp_pkey + self._key_size_bits = _get_dh_num_bits(self._backend, self._dh_cdata) + + @property + def key_size(self): + return self._key_size_bits + + def public_numbers(self): + p = self._backend._ffi.new("BIGNUM **") + g = self._backend._ffi.new("BIGNUM **") + q = self._backend._ffi.new("BIGNUM **") + self._backend._lib.DH_get0_pqg(self._dh_cdata, p, q, g) + self._backend.openssl_assert(p[0] != self._backend._ffi.NULL) + self._backend.openssl_assert(g[0] != self._backend._ffi.NULL) + if q[0] == self._backend._ffi.NULL: + q_val = None + else: + q_val = self._backend._bn_to_int(q[0]) + pub_key = self._backend._ffi.new("BIGNUM **") + self._backend._lib.DH_get0_key( + self._dh_cdata, pub_key, self._backend._ffi.NULL + ) + self._backend.openssl_assert(pub_key[0] != self._backend._ffi.NULL) + return dh.DHPublicNumbers( + parameter_numbers=dh.DHParameterNumbers( + p=self._backend._bn_to_int(p[0]), + g=self._backend._bn_to_int(g[0]), + q=q_val, + ), + y=self._backend._bn_to_int(pub_key[0]), + ) + + def parameters(self): + return _dh_cdata_to_parameters(self._dh_cdata, self._backend) + + def public_bytes(self, encoding, format): + if format is not serialization.PublicFormat.SubjectPublicKeyInfo: + raise ValueError( + "DH public keys support only " + "SubjectPublicKeyInfo serialization" + ) + + if not self._backend._lib.Cryptography_HAS_EVP_PKEY_DHX: + q = self._backend._ffi.new("BIGNUM **") + self._backend._lib.DH_get0_pqg( + self._dh_cdata, + self._backend._ffi.NULL, + q, + self._backend._ffi.NULL, + ) + if q[0] != self._backend._ffi.NULL: + raise UnsupportedAlgorithm( + "DH X9.42 serialization is not supported", + _Reasons.UNSUPPORTED_SERIALIZATION, + ) + + return self._backend._public_key_bytes( + encoding, format, self, self._evp_pkey, None + ) diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/openssl/dsa.py b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/openssl/dsa.py new file mode 100644 index 0000000..0c5faba --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/openssl/dsa.py @@ -0,0 +1,263 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function + +from cryptography import utils +from cryptography.exceptions import InvalidSignature +from cryptography.hazmat.backends.openssl.utils import ( + _calculate_digest_and_algorithm, + _check_not_prehashed, + _warn_sign_verify_deprecated, +) +from cryptography.hazmat.primitives import hashes +from cryptography.hazmat.primitives.asymmetric import ( + AsymmetricSignatureContext, + AsymmetricVerificationContext, + dsa, +) + + +def _dsa_sig_sign(backend, private_key, data): + sig_buf_len = backend._lib.DSA_size(private_key._dsa_cdata) + sig_buf = backend._ffi.new("unsigned char[]", sig_buf_len) + buflen = backend._ffi.new("unsigned int *") + + # The first parameter passed to DSA_sign is unused by OpenSSL but + # must be an integer. + res = backend._lib.DSA_sign( + 0, data, len(data), sig_buf, buflen, private_key._dsa_cdata + ) + backend.openssl_assert(res == 1) + backend.openssl_assert(buflen[0]) + + return backend._ffi.buffer(sig_buf)[: buflen[0]] + + +def _dsa_sig_verify(backend, public_key, signature, data): + # The first parameter passed to DSA_verify is unused by OpenSSL but + # must be an integer. + res = backend._lib.DSA_verify( + 0, data, len(data), signature, len(signature), public_key._dsa_cdata + ) + + if res != 1: + backend._consume_errors() + raise InvalidSignature + + +@utils.register_interface(AsymmetricVerificationContext) +class _DSAVerificationContext(object): + def __init__(self, backend, public_key, signature, algorithm): + self._backend = backend + self._public_key = public_key + self._signature = signature + self._algorithm = algorithm + + self._hash_ctx = hashes.Hash(self._algorithm, self._backend) + + def update(self, data): + self._hash_ctx.update(data) + + def verify(self): + data_to_verify = self._hash_ctx.finalize() + + _dsa_sig_verify( + self._backend, self._public_key, self._signature, data_to_verify + ) + + +@utils.register_interface(AsymmetricSignatureContext) +class _DSASignatureContext(object): + def __init__(self, backend, private_key, algorithm): + self._backend = backend + self._private_key = private_key + self._algorithm = algorithm + self._hash_ctx = hashes.Hash(self._algorithm, self._backend) + + def update(self, data): + self._hash_ctx.update(data) + + def finalize(self): + data_to_sign = self._hash_ctx.finalize() + return _dsa_sig_sign(self._backend, self._private_key, data_to_sign) + + +@utils.register_interface(dsa.DSAParametersWithNumbers) +class _DSAParameters(object): + def __init__(self, backend, dsa_cdata): + self._backend = backend + self._dsa_cdata = dsa_cdata + + def parameter_numbers(self): + p = self._backend._ffi.new("BIGNUM **") + q = self._backend._ffi.new("BIGNUM **") + g = self._backend._ffi.new("BIGNUM **") + self._backend._lib.DSA_get0_pqg(self._dsa_cdata, p, q, g) + self._backend.openssl_assert(p[0] != self._backend._ffi.NULL) + self._backend.openssl_assert(q[0] != self._backend._ffi.NULL) + self._backend.openssl_assert(g[0] != self._backend._ffi.NULL) + return dsa.DSAParameterNumbers( + p=self._backend._bn_to_int(p[0]), + q=self._backend._bn_to_int(q[0]), + g=self._backend._bn_to_int(g[0]), + ) + + def generate_private_key(self): + return self._backend.generate_dsa_private_key(self) + + +@utils.register_interface(dsa.DSAPrivateKeyWithSerialization) +class _DSAPrivateKey(object): + def __init__(self, backend, dsa_cdata, evp_pkey): + self._backend = backend + self._dsa_cdata = dsa_cdata + self._evp_pkey = evp_pkey + + p = self._backend._ffi.new("BIGNUM **") + self._backend._lib.DSA_get0_pqg( + dsa_cdata, p, self._backend._ffi.NULL, self._backend._ffi.NULL + ) + self._backend.openssl_assert(p[0] != backend._ffi.NULL) + self._key_size = self._backend._lib.BN_num_bits(p[0]) + + key_size = utils.read_only_property("_key_size") + + def signer(self, signature_algorithm): + _warn_sign_verify_deprecated() + _check_not_prehashed(signature_algorithm) + return _DSASignatureContext(self._backend, self, signature_algorithm) + + def private_numbers(self): + p = self._backend._ffi.new("BIGNUM **") + q = self._backend._ffi.new("BIGNUM **") + g = self._backend._ffi.new("BIGNUM **") + pub_key = self._backend._ffi.new("BIGNUM **") + priv_key = self._backend._ffi.new("BIGNUM **") + self._backend._lib.DSA_get0_pqg(self._dsa_cdata, p, q, g) + self._backend.openssl_assert(p[0] != self._backend._ffi.NULL) + self._backend.openssl_assert(q[0] != self._backend._ffi.NULL) + self._backend.openssl_assert(g[0] != self._backend._ffi.NULL) + self._backend._lib.DSA_get0_key(self._dsa_cdata, pub_key, priv_key) + self._backend.openssl_assert(pub_key[0] != self._backend._ffi.NULL) + self._backend.openssl_assert(priv_key[0] != self._backend._ffi.NULL) + return dsa.DSAPrivateNumbers( + public_numbers=dsa.DSAPublicNumbers( + parameter_numbers=dsa.DSAParameterNumbers( + p=self._backend._bn_to_int(p[0]), + q=self._backend._bn_to_int(q[0]), + g=self._backend._bn_to_int(g[0]), + ), + y=self._backend._bn_to_int(pub_key[0]), + ), + x=self._backend._bn_to_int(priv_key[0]), + ) + + def public_key(self): + dsa_cdata = self._backend._lib.DSAparams_dup(self._dsa_cdata) + self._backend.openssl_assert(dsa_cdata != self._backend._ffi.NULL) + dsa_cdata = self._backend._ffi.gc( + dsa_cdata, self._backend._lib.DSA_free + ) + pub_key = self._backend._ffi.new("BIGNUM **") + self._backend._lib.DSA_get0_key( + self._dsa_cdata, pub_key, self._backend._ffi.NULL + ) + self._backend.openssl_assert(pub_key[0] != self._backend._ffi.NULL) + pub_key_dup = self._backend._lib.BN_dup(pub_key[0]) + res = self._backend._lib.DSA_set0_key( + dsa_cdata, pub_key_dup, self._backend._ffi.NULL + ) + self._backend.openssl_assert(res == 1) + evp_pkey = self._backend._dsa_cdata_to_evp_pkey(dsa_cdata) + return _DSAPublicKey(self._backend, dsa_cdata, evp_pkey) + + def parameters(self): + dsa_cdata = self._backend._lib.DSAparams_dup(self._dsa_cdata) + self._backend.openssl_assert(dsa_cdata != self._backend._ffi.NULL) + dsa_cdata = self._backend._ffi.gc( + dsa_cdata, self._backend._lib.DSA_free + ) + return _DSAParameters(self._backend, dsa_cdata) + + def private_bytes(self, encoding, format, encryption_algorithm): + return self._backend._private_key_bytes( + encoding, + format, + encryption_algorithm, + self, + self._evp_pkey, + self._dsa_cdata, + ) + + def sign(self, data, algorithm): + data, algorithm = _calculate_digest_and_algorithm( + self._backend, data, algorithm + ) + return _dsa_sig_sign(self._backend, self, data) + + +@utils.register_interface(dsa.DSAPublicKeyWithSerialization) +class _DSAPublicKey(object): + def __init__(self, backend, dsa_cdata, evp_pkey): + self._backend = backend + self._dsa_cdata = dsa_cdata + self._evp_pkey = evp_pkey + p = self._backend._ffi.new("BIGNUM **") + self._backend._lib.DSA_get0_pqg( + dsa_cdata, p, self._backend._ffi.NULL, self._backend._ffi.NULL + ) + self._backend.openssl_assert(p[0] != backend._ffi.NULL) + self._key_size = self._backend._lib.BN_num_bits(p[0]) + + key_size = utils.read_only_property("_key_size") + + def verifier(self, signature, signature_algorithm): + _warn_sign_verify_deprecated() + utils._check_bytes("signature", signature) + + _check_not_prehashed(signature_algorithm) + return _DSAVerificationContext( + self._backend, self, signature, signature_algorithm + ) + + def public_numbers(self): + p = self._backend._ffi.new("BIGNUM **") + q = self._backend._ffi.new("BIGNUM **") + g = self._backend._ffi.new("BIGNUM **") + pub_key = self._backend._ffi.new("BIGNUM **") + self._backend._lib.DSA_get0_pqg(self._dsa_cdata, p, q, g) + self._backend.openssl_assert(p[0] != self._backend._ffi.NULL) + self._backend.openssl_assert(q[0] != self._backend._ffi.NULL) + self._backend.openssl_assert(g[0] != self._backend._ffi.NULL) + self._backend._lib.DSA_get0_key( + self._dsa_cdata, pub_key, self._backend._ffi.NULL + ) + self._backend.openssl_assert(pub_key[0] != self._backend._ffi.NULL) + return dsa.DSAPublicNumbers( + parameter_numbers=dsa.DSAParameterNumbers( + p=self._backend._bn_to_int(p[0]), + q=self._backend._bn_to_int(q[0]), + g=self._backend._bn_to_int(g[0]), + ), + y=self._backend._bn_to_int(pub_key[0]), + ) + + def parameters(self): + dsa_cdata = self._backend._lib.DSAparams_dup(self._dsa_cdata) + dsa_cdata = self._backend._ffi.gc( + dsa_cdata, self._backend._lib.DSA_free + ) + return _DSAParameters(self._backend, dsa_cdata) + + def public_bytes(self, encoding, format): + return self._backend._public_key_bytes( + encoding, format, self, self._evp_pkey, None + ) + + def verify(self, signature, data, algorithm): + data, algorithm = _calculate_digest_and_algorithm( + self._backend, data, algorithm + ) + return _dsa_sig_verify(self._backend, self, signature, data) diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/openssl/ec.py b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/openssl/ec.py new file mode 100644 index 0000000..05d32ba --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/openssl/ec.py @@ -0,0 +1,337 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function + +from cryptography import utils +from cryptography.exceptions import ( + InvalidSignature, + UnsupportedAlgorithm, + _Reasons, +) +from cryptography.hazmat.backends.openssl.utils import ( + _calculate_digest_and_algorithm, + _check_not_prehashed, + _warn_sign_verify_deprecated, +) +from cryptography.hazmat.primitives import hashes, serialization +from cryptography.hazmat.primitives.asymmetric import ( + AsymmetricSignatureContext, + AsymmetricVerificationContext, + ec, +) + + +def _check_signature_algorithm(signature_algorithm): + if not isinstance(signature_algorithm, ec.ECDSA): + raise UnsupportedAlgorithm( + "Unsupported elliptic curve signature algorithm.", + _Reasons.UNSUPPORTED_PUBLIC_KEY_ALGORITHM, + ) + + +def _ec_key_curve_sn(backend, ec_key): + group = backend._lib.EC_KEY_get0_group(ec_key) + backend.openssl_assert(group != backend._ffi.NULL) + + nid = backend._lib.EC_GROUP_get_curve_name(group) + # The following check is to find EC keys with unnamed curves and raise + # an error for now. + if nid == backend._lib.NID_undef: + raise NotImplementedError( + "ECDSA keys with unnamed curves are unsupported at this time" + ) + + # This is like the above check, but it also catches the case where you + # explicitly encoded a curve with the same parameters as a named curve. + # Don't do that. + if ( + not backend._lib.CRYPTOGRAPHY_IS_LIBRESSL + and backend._lib.EC_GROUP_get_asn1_flag(group) == 0 + ): + raise NotImplementedError( + "ECDSA keys with unnamed curves are unsupported at this time" + ) + + curve_name = backend._lib.OBJ_nid2sn(nid) + backend.openssl_assert(curve_name != backend._ffi.NULL) + + sn = backend._ffi.string(curve_name).decode("ascii") + return sn + + +def _mark_asn1_named_ec_curve(backend, ec_cdata): + """ + Set the named curve flag on the EC_KEY. This causes OpenSSL to + serialize EC keys along with their curve OID which makes + deserialization easier. + """ + + backend._lib.EC_KEY_set_asn1_flag( + ec_cdata, backend._lib.OPENSSL_EC_NAMED_CURVE + ) + + +def _sn_to_elliptic_curve(backend, sn): + try: + return ec._CURVE_TYPES[sn]() + except KeyError: + raise UnsupportedAlgorithm( + "{} is not a supported elliptic curve".format(sn), + _Reasons.UNSUPPORTED_ELLIPTIC_CURVE, + ) + + +def _ecdsa_sig_sign(backend, private_key, data): + max_size = backend._lib.ECDSA_size(private_key._ec_key) + backend.openssl_assert(max_size > 0) + + sigbuf = backend._ffi.new("unsigned char[]", max_size) + siglen_ptr = backend._ffi.new("unsigned int[]", 1) + res = backend._lib.ECDSA_sign( + 0, data, len(data), sigbuf, siglen_ptr, private_key._ec_key + ) + backend.openssl_assert(res == 1) + return backend._ffi.buffer(sigbuf)[: siglen_ptr[0]] + + +def _ecdsa_sig_verify(backend, public_key, signature, data): + res = backend._lib.ECDSA_verify( + 0, data, len(data), signature, len(signature), public_key._ec_key + ) + if res != 1: + backend._consume_errors() + raise InvalidSignature + + +@utils.register_interface(AsymmetricSignatureContext) +class _ECDSASignatureContext(object): + def __init__(self, backend, private_key, algorithm): + self._backend = backend + self._private_key = private_key + self._digest = hashes.Hash(algorithm, backend) + + def update(self, data): + self._digest.update(data) + + def finalize(self): + digest = self._digest.finalize() + + return _ecdsa_sig_sign(self._backend, self._private_key, digest) + + +@utils.register_interface(AsymmetricVerificationContext) +class _ECDSAVerificationContext(object): + def __init__(self, backend, public_key, signature, algorithm): + self._backend = backend + self._public_key = public_key + self._signature = signature + self._digest = hashes.Hash(algorithm, backend) + + def update(self, data): + self._digest.update(data) + + def verify(self): + digest = self._digest.finalize() + _ecdsa_sig_verify( + self._backend, self._public_key, self._signature, digest + ) + + +@utils.register_interface(ec.EllipticCurvePrivateKeyWithSerialization) +class _EllipticCurvePrivateKey(object): + def __init__(self, backend, ec_key_cdata, evp_pkey): + self._backend = backend + self._ec_key = ec_key_cdata + self._evp_pkey = evp_pkey + + sn = _ec_key_curve_sn(backend, ec_key_cdata) + self._curve = _sn_to_elliptic_curve(backend, sn) + _mark_asn1_named_ec_curve(backend, ec_key_cdata) + + curve = utils.read_only_property("_curve") + + @property + def key_size(self): + return self.curve.key_size + + def signer(self, signature_algorithm): + _warn_sign_verify_deprecated() + _check_signature_algorithm(signature_algorithm) + _check_not_prehashed(signature_algorithm.algorithm) + return _ECDSASignatureContext( + self._backend, self, signature_algorithm.algorithm + ) + + def exchange(self, algorithm, peer_public_key): + if not ( + self._backend.elliptic_curve_exchange_algorithm_supported( + algorithm, self.curve + ) + ): + raise UnsupportedAlgorithm( + "This backend does not support the ECDH algorithm.", + _Reasons.UNSUPPORTED_EXCHANGE_ALGORITHM, + ) + + if peer_public_key.curve.name != self.curve.name: + raise ValueError( + "peer_public_key and self are not on the same curve" + ) + + group = self._backend._lib.EC_KEY_get0_group(self._ec_key) + z_len = (self._backend._lib.EC_GROUP_get_degree(group) + 7) // 8 + self._backend.openssl_assert(z_len > 0) + z_buf = self._backend._ffi.new("uint8_t[]", z_len) + peer_key = self._backend._lib.EC_KEY_get0_public_key( + peer_public_key._ec_key + ) + + r = self._backend._lib.ECDH_compute_key( + z_buf, z_len, peer_key, self._ec_key, self._backend._ffi.NULL + ) + self._backend.openssl_assert(r > 0) + return self._backend._ffi.buffer(z_buf)[:z_len] + + def public_key(self): + group = self._backend._lib.EC_KEY_get0_group(self._ec_key) + self._backend.openssl_assert(group != self._backend._ffi.NULL) + + curve_nid = self._backend._lib.EC_GROUP_get_curve_name(group) + public_ec_key = self._backend._ec_key_new_by_curve_nid(curve_nid) + + point = self._backend._lib.EC_KEY_get0_public_key(self._ec_key) + self._backend.openssl_assert(point != self._backend._ffi.NULL) + + res = self._backend._lib.EC_KEY_set_public_key(public_ec_key, point) + self._backend.openssl_assert(res == 1) + + evp_pkey = self._backend._ec_cdata_to_evp_pkey(public_ec_key) + + return _EllipticCurvePublicKey(self._backend, public_ec_key, evp_pkey) + + def private_numbers(self): + bn = self._backend._lib.EC_KEY_get0_private_key(self._ec_key) + private_value = self._backend._bn_to_int(bn) + return ec.EllipticCurvePrivateNumbers( + private_value=private_value, + public_numbers=self.public_key().public_numbers(), + ) + + def private_bytes(self, encoding, format, encryption_algorithm): + return self._backend._private_key_bytes( + encoding, + format, + encryption_algorithm, + self, + self._evp_pkey, + self._ec_key, + ) + + def sign(self, data, signature_algorithm): + _check_signature_algorithm(signature_algorithm) + data, algorithm = _calculate_digest_and_algorithm( + self._backend, data, signature_algorithm._algorithm + ) + return _ecdsa_sig_sign(self._backend, self, data) + + +@utils.register_interface(ec.EllipticCurvePublicKeyWithSerialization) +class _EllipticCurvePublicKey(object): + def __init__(self, backend, ec_key_cdata, evp_pkey): + self._backend = backend + self._ec_key = ec_key_cdata + self._evp_pkey = evp_pkey + + sn = _ec_key_curve_sn(backend, ec_key_cdata) + self._curve = _sn_to_elliptic_curve(backend, sn) + _mark_asn1_named_ec_curve(backend, ec_key_cdata) + + curve = utils.read_only_property("_curve") + + @property + def key_size(self): + return self.curve.key_size + + def verifier(self, signature, signature_algorithm): + _warn_sign_verify_deprecated() + utils._check_bytes("signature", signature) + + _check_signature_algorithm(signature_algorithm) + _check_not_prehashed(signature_algorithm.algorithm) + return _ECDSAVerificationContext( + self._backend, self, signature, signature_algorithm.algorithm + ) + + def public_numbers(self): + get_func, group = self._backend._ec_key_determine_group_get_func( + self._ec_key + ) + point = self._backend._lib.EC_KEY_get0_public_key(self._ec_key) + self._backend.openssl_assert(point != self._backend._ffi.NULL) + + with self._backend._tmp_bn_ctx() as bn_ctx: + bn_x = self._backend._lib.BN_CTX_get(bn_ctx) + bn_y = self._backend._lib.BN_CTX_get(bn_ctx) + + res = get_func(group, point, bn_x, bn_y, bn_ctx) + self._backend.openssl_assert(res == 1) + + x = self._backend._bn_to_int(bn_x) + y = self._backend._bn_to_int(bn_y) + + return ec.EllipticCurvePublicNumbers(x=x, y=y, curve=self._curve) + + def _encode_point(self, format): + if format is serialization.PublicFormat.CompressedPoint: + conversion = self._backend._lib.POINT_CONVERSION_COMPRESSED + else: + assert format is serialization.PublicFormat.UncompressedPoint + conversion = self._backend._lib.POINT_CONVERSION_UNCOMPRESSED + + group = self._backend._lib.EC_KEY_get0_group(self._ec_key) + self._backend.openssl_assert(group != self._backend._ffi.NULL) + point = self._backend._lib.EC_KEY_get0_public_key(self._ec_key) + self._backend.openssl_assert(point != self._backend._ffi.NULL) + with self._backend._tmp_bn_ctx() as bn_ctx: + buflen = self._backend._lib.EC_POINT_point2oct( + group, point, conversion, self._backend._ffi.NULL, 0, bn_ctx + ) + self._backend.openssl_assert(buflen > 0) + buf = self._backend._ffi.new("char[]", buflen) + res = self._backend._lib.EC_POINT_point2oct( + group, point, conversion, buf, buflen, bn_ctx + ) + self._backend.openssl_assert(buflen == res) + + return self._backend._ffi.buffer(buf)[:] + + def public_bytes(self, encoding, format): + + if ( + encoding is serialization.Encoding.X962 + or format is serialization.PublicFormat.CompressedPoint + or format is serialization.PublicFormat.UncompressedPoint + ): + if encoding is not serialization.Encoding.X962 or format not in ( + serialization.PublicFormat.CompressedPoint, + serialization.PublicFormat.UncompressedPoint, + ): + raise ValueError( + "X962 encoding must be used with CompressedPoint or " + "UncompressedPoint format" + ) + + return self._encode_point(format) + else: + return self._backend._public_key_bytes( + encoding, format, self, self._evp_pkey, None + ) + + def verify(self, signature, data, signature_algorithm): + _check_signature_algorithm(signature_algorithm) + data, algorithm = _calculate_digest_and_algorithm( + self._backend, data, signature_algorithm._algorithm + ) + _ecdsa_sig_verify(self._backend, self, signature, data) diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/openssl/ed25519.py b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/openssl/ed25519.py new file mode 100644 index 0000000..13bec3a --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/openssl/ed25519.py @@ -0,0 +1,145 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function + +from cryptography import exceptions, utils +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.asymmetric.ed25519 import ( + Ed25519PrivateKey, + Ed25519PublicKey, + _ED25519_KEY_SIZE, + _ED25519_SIG_SIZE, +) + + +@utils.register_interface(Ed25519PublicKey) +class _Ed25519PublicKey(object): + def __init__(self, backend, evp_pkey): + self._backend = backend + self._evp_pkey = evp_pkey + + def public_bytes(self, encoding, format): + if ( + encoding is serialization.Encoding.Raw + or format is serialization.PublicFormat.Raw + ): + if ( + encoding is not serialization.Encoding.Raw + or format is not serialization.PublicFormat.Raw + ): + raise ValueError( + "When using Raw both encoding and format must be Raw" + ) + + return self._raw_public_bytes() + + return self._backend._public_key_bytes( + encoding, format, self, self._evp_pkey, None + ) + + def _raw_public_bytes(self): + buf = self._backend._ffi.new("unsigned char []", _ED25519_KEY_SIZE) + buflen = self._backend._ffi.new("size_t *", _ED25519_KEY_SIZE) + res = self._backend._lib.EVP_PKEY_get_raw_public_key( + self._evp_pkey, buf, buflen + ) + self._backend.openssl_assert(res == 1) + self._backend.openssl_assert(buflen[0] == _ED25519_KEY_SIZE) + return self._backend._ffi.buffer(buf, _ED25519_KEY_SIZE)[:] + + def verify(self, signature, data): + evp_md_ctx = self._backend._lib.EVP_MD_CTX_new() + self._backend.openssl_assert(evp_md_ctx != self._backend._ffi.NULL) + evp_md_ctx = self._backend._ffi.gc( + evp_md_ctx, self._backend._lib.EVP_MD_CTX_free + ) + res = self._backend._lib.EVP_DigestVerifyInit( + evp_md_ctx, + self._backend._ffi.NULL, + self._backend._ffi.NULL, + self._backend._ffi.NULL, + self._evp_pkey, + ) + self._backend.openssl_assert(res == 1) + res = self._backend._lib.EVP_DigestVerify( + evp_md_ctx, signature, len(signature), data, len(data) + ) + if res != 1: + self._backend._consume_errors() + raise exceptions.InvalidSignature + + +@utils.register_interface(Ed25519PrivateKey) +class _Ed25519PrivateKey(object): + def __init__(self, backend, evp_pkey): + self._backend = backend + self._evp_pkey = evp_pkey + + def public_key(self): + buf = self._backend._ffi.new("unsigned char []", _ED25519_KEY_SIZE) + buflen = self._backend._ffi.new("size_t *", _ED25519_KEY_SIZE) + res = self._backend._lib.EVP_PKEY_get_raw_public_key( + self._evp_pkey, buf, buflen + ) + self._backend.openssl_assert(res == 1) + self._backend.openssl_assert(buflen[0] == _ED25519_KEY_SIZE) + public_bytes = self._backend._ffi.buffer(buf)[:] + return self._backend.ed25519_load_public_bytes(public_bytes) + + def sign(self, data): + evp_md_ctx = self._backend._lib.EVP_MD_CTX_new() + self._backend.openssl_assert(evp_md_ctx != self._backend._ffi.NULL) + evp_md_ctx = self._backend._ffi.gc( + evp_md_ctx, self._backend._lib.EVP_MD_CTX_free + ) + res = self._backend._lib.EVP_DigestSignInit( + evp_md_ctx, + self._backend._ffi.NULL, + self._backend._ffi.NULL, + self._backend._ffi.NULL, + self._evp_pkey, + ) + self._backend.openssl_assert(res == 1) + buf = self._backend._ffi.new("unsigned char[]", _ED25519_SIG_SIZE) + buflen = self._backend._ffi.new("size_t *", len(buf)) + res = self._backend._lib.EVP_DigestSign( + evp_md_ctx, buf, buflen, data, len(data) + ) + self._backend.openssl_assert(res == 1) + self._backend.openssl_assert(buflen[0] == _ED25519_SIG_SIZE) + return self._backend._ffi.buffer(buf, buflen[0])[:] + + def private_bytes(self, encoding, format, encryption_algorithm): + if ( + encoding is serialization.Encoding.Raw + or format is serialization.PublicFormat.Raw + ): + if ( + format is not serialization.PrivateFormat.Raw + or encoding is not serialization.Encoding.Raw + or not isinstance( + encryption_algorithm, serialization.NoEncryption + ) + ): + raise ValueError( + "When using Raw both encoding and format must be Raw " + "and encryption_algorithm must be NoEncryption()" + ) + + return self._raw_private_bytes() + + return self._backend._private_key_bytes( + encoding, format, encryption_algorithm, self, self._evp_pkey, None + ) + + def _raw_private_bytes(self): + buf = self._backend._ffi.new("unsigned char []", _ED25519_KEY_SIZE) + buflen = self._backend._ffi.new("size_t *", _ED25519_KEY_SIZE) + res = self._backend._lib.EVP_PKEY_get_raw_private_key( + self._evp_pkey, buf, buflen + ) + self._backend.openssl_assert(res == 1) + self._backend.openssl_assert(buflen[0] == _ED25519_KEY_SIZE) + return self._backend._ffi.buffer(buf, _ED25519_KEY_SIZE)[:] diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/openssl/ed448.py b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/openssl/ed448.py new file mode 100644 index 0000000..6512770 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/openssl/ed448.py @@ -0,0 +1,146 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function + +from cryptography import exceptions, utils +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.asymmetric.ed448 import ( + Ed448PrivateKey, + Ed448PublicKey, +) + +_ED448_KEY_SIZE = 57 +_ED448_SIG_SIZE = 114 + + +@utils.register_interface(Ed448PublicKey) +class _Ed448PublicKey(object): + def __init__(self, backend, evp_pkey): + self._backend = backend + self._evp_pkey = evp_pkey + + def public_bytes(self, encoding, format): + if ( + encoding is serialization.Encoding.Raw + or format is serialization.PublicFormat.Raw + ): + if ( + encoding is not serialization.Encoding.Raw + or format is not serialization.PublicFormat.Raw + ): + raise ValueError( + "When using Raw both encoding and format must be Raw" + ) + + return self._raw_public_bytes() + + return self._backend._public_key_bytes( + encoding, format, self, self._evp_pkey, None + ) + + def _raw_public_bytes(self): + buf = self._backend._ffi.new("unsigned char []", _ED448_KEY_SIZE) + buflen = self._backend._ffi.new("size_t *", _ED448_KEY_SIZE) + res = self._backend._lib.EVP_PKEY_get_raw_public_key( + self._evp_pkey, buf, buflen + ) + self._backend.openssl_assert(res == 1) + self._backend.openssl_assert(buflen[0] == _ED448_KEY_SIZE) + return self._backend._ffi.buffer(buf, _ED448_KEY_SIZE)[:] + + def verify(self, signature, data): + evp_md_ctx = self._backend._lib.EVP_MD_CTX_new() + self._backend.openssl_assert(evp_md_ctx != self._backend._ffi.NULL) + evp_md_ctx = self._backend._ffi.gc( + evp_md_ctx, self._backend._lib.EVP_MD_CTX_free + ) + res = self._backend._lib.EVP_DigestVerifyInit( + evp_md_ctx, + self._backend._ffi.NULL, + self._backend._ffi.NULL, + self._backend._ffi.NULL, + self._evp_pkey, + ) + self._backend.openssl_assert(res == 1) + res = self._backend._lib.EVP_DigestVerify( + evp_md_ctx, signature, len(signature), data, len(data) + ) + if res != 1: + self._backend._consume_errors() + raise exceptions.InvalidSignature + + +@utils.register_interface(Ed448PrivateKey) +class _Ed448PrivateKey(object): + def __init__(self, backend, evp_pkey): + self._backend = backend + self._evp_pkey = evp_pkey + + def public_key(self): + buf = self._backend._ffi.new("unsigned char []", _ED448_KEY_SIZE) + buflen = self._backend._ffi.new("size_t *", _ED448_KEY_SIZE) + res = self._backend._lib.EVP_PKEY_get_raw_public_key( + self._evp_pkey, buf, buflen + ) + self._backend.openssl_assert(res == 1) + self._backend.openssl_assert(buflen[0] == _ED448_KEY_SIZE) + public_bytes = self._backend._ffi.buffer(buf)[:] + return self._backend.ed448_load_public_bytes(public_bytes) + + def sign(self, data): + evp_md_ctx = self._backend._lib.EVP_MD_CTX_new() + self._backend.openssl_assert(evp_md_ctx != self._backend._ffi.NULL) + evp_md_ctx = self._backend._ffi.gc( + evp_md_ctx, self._backend._lib.EVP_MD_CTX_free + ) + res = self._backend._lib.EVP_DigestSignInit( + evp_md_ctx, + self._backend._ffi.NULL, + self._backend._ffi.NULL, + self._backend._ffi.NULL, + self._evp_pkey, + ) + self._backend.openssl_assert(res == 1) + buf = self._backend._ffi.new("unsigned char[]", _ED448_SIG_SIZE) + buflen = self._backend._ffi.new("size_t *", len(buf)) + res = self._backend._lib.EVP_DigestSign( + evp_md_ctx, buf, buflen, data, len(data) + ) + self._backend.openssl_assert(res == 1) + self._backend.openssl_assert(buflen[0] == _ED448_SIG_SIZE) + return self._backend._ffi.buffer(buf, buflen[0])[:] + + def private_bytes(self, encoding, format, encryption_algorithm): + if ( + encoding is serialization.Encoding.Raw + or format is serialization.PublicFormat.Raw + ): + if ( + format is not serialization.PrivateFormat.Raw + or encoding is not serialization.Encoding.Raw + or not isinstance( + encryption_algorithm, serialization.NoEncryption + ) + ): + raise ValueError( + "When using Raw both encoding and format must be Raw " + "and encryption_algorithm must be NoEncryption()" + ) + + return self._raw_private_bytes() + + return self._backend._private_key_bytes( + encoding, format, encryption_algorithm, self, self._evp_pkey, None + ) + + def _raw_private_bytes(self): + buf = self._backend._ffi.new("unsigned char []", _ED448_KEY_SIZE) + buflen = self._backend._ffi.new("size_t *", _ED448_KEY_SIZE) + res = self._backend._lib.EVP_PKEY_get_raw_private_key( + self._evp_pkey, buf, buflen + ) + self._backend.openssl_assert(res == 1) + self._backend.openssl_assert(buflen[0] == _ED448_KEY_SIZE) + return self._backend._ffi.buffer(buf, _ED448_KEY_SIZE)[:] diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/openssl/encode_asn1.py b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/openssl/encode_asn1.py new file mode 100644 index 0000000..0a33200 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/openssl/encode_asn1.py @@ -0,0 +1,670 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function + +import calendar +import ipaddress + +import six + +from cryptography import utils, x509 +from cryptography.hazmat.backends.openssl.decode_asn1 import ( + _CRL_ENTRY_REASON_ENUM_TO_CODE, + _DISTPOINT_TYPE_FULLNAME, + _DISTPOINT_TYPE_RELATIVENAME, +) +from cryptography.x509.name import _ASN1Type +from cryptography.x509.oid import ( + CRLEntryExtensionOID, + ExtensionOID, + OCSPExtensionOID, +) + + +def _encode_asn1_int(backend, x): + """ + Converts a python integer to an ASN1_INTEGER. The returned ASN1_INTEGER + will not be garbage collected (to support adding them to structs that take + ownership of the object). Be sure to register it for GC if it will be + discarded after use. + + """ + # Convert Python integer to OpenSSL "bignum" in case value exceeds + # machine's native integer limits (note: `int_to_bn` doesn't automatically + # GC). + i = backend._int_to_bn(x) + i = backend._ffi.gc(i, backend._lib.BN_free) + + # Wrap in an ASN.1 integer. Don't GC -- as documented. + i = backend._lib.BN_to_ASN1_INTEGER(i, backend._ffi.NULL) + backend.openssl_assert(i != backend._ffi.NULL) + return i + + +def _encode_asn1_int_gc(backend, x): + i = _encode_asn1_int(backend, x) + i = backend._ffi.gc(i, backend._lib.ASN1_INTEGER_free) + return i + + +def _encode_asn1_str(backend, data): + """ + Create an ASN1_OCTET_STRING from a Python byte string. + """ + s = backend._lib.ASN1_OCTET_STRING_new() + res = backend._lib.ASN1_OCTET_STRING_set(s, data, len(data)) + backend.openssl_assert(res == 1) + return s + + +def _encode_asn1_utf8_str(backend, string): + """ + Create an ASN1_UTF8STRING from a Python unicode string. + This object will be an ASN1_STRING with UTF8 type in OpenSSL and + can be decoded with ASN1_STRING_to_UTF8. + """ + s = backend._lib.ASN1_UTF8STRING_new() + res = backend._lib.ASN1_STRING_set( + s, string.encode("utf8"), len(string.encode("utf8")) + ) + backend.openssl_assert(res == 1) + return s + + +def _encode_asn1_str_gc(backend, data): + s = _encode_asn1_str(backend, data) + s = backend._ffi.gc(s, backend._lib.ASN1_OCTET_STRING_free) + return s + + +def _encode_inhibit_any_policy(backend, inhibit_any_policy): + return _encode_asn1_int_gc(backend, inhibit_any_policy.skip_certs) + + +def _encode_name(backend, name): + """ + The X509_NAME created will not be gc'd. Use _encode_name_gc if needed. + """ + subject = backend._lib.X509_NAME_new() + for rdn in name.rdns: + set_flag = 0 # indicate whether to add to last RDN or create new RDN + for attribute in rdn: + name_entry = _encode_name_entry(backend, attribute) + # X509_NAME_add_entry dups the object so we need to gc this copy + name_entry = backend._ffi.gc( + name_entry, backend._lib.X509_NAME_ENTRY_free + ) + res = backend._lib.X509_NAME_add_entry( + subject, name_entry, -1, set_flag + ) + backend.openssl_assert(res == 1) + set_flag = -1 + return subject + + +def _encode_name_gc(backend, attributes): + subject = _encode_name(backend, attributes) + subject = backend._ffi.gc(subject, backend._lib.X509_NAME_free) + return subject + + +def _encode_sk_name_entry(backend, attributes): + """ + The sk_X509_NAME_ENTRY created will not be gc'd. + """ + stack = backend._lib.sk_X509_NAME_ENTRY_new_null() + for attribute in attributes: + name_entry = _encode_name_entry(backend, attribute) + res = backend._lib.sk_X509_NAME_ENTRY_push(stack, name_entry) + backend.openssl_assert(res >= 1) + return stack + + +def _encode_name_entry(backend, attribute): + if attribute._type is _ASN1Type.BMPString: + value = attribute.value.encode("utf_16_be") + elif attribute._type is _ASN1Type.UniversalString: + value = attribute.value.encode("utf_32_be") + else: + value = attribute.value.encode("utf8") + + obj = _txt2obj_gc(backend, attribute.oid.dotted_string) + + name_entry = backend._lib.X509_NAME_ENTRY_create_by_OBJ( + backend._ffi.NULL, obj, attribute._type.value, value, len(value) + ) + return name_entry + + +def _encode_crl_number_delta_crl_indicator(backend, ext): + return _encode_asn1_int_gc(backend, ext.crl_number) + + +def _encode_issuing_dist_point(backend, ext): + idp = backend._lib.ISSUING_DIST_POINT_new() + backend.openssl_assert(idp != backend._ffi.NULL) + idp = backend._ffi.gc(idp, backend._lib.ISSUING_DIST_POINT_free) + idp.onlyuser = 255 if ext.only_contains_user_certs else 0 + idp.onlyCA = 255 if ext.only_contains_ca_certs else 0 + idp.indirectCRL = 255 if ext.indirect_crl else 0 + idp.onlyattr = 255 if ext.only_contains_attribute_certs else 0 + if ext.only_some_reasons: + idp.onlysomereasons = _encode_reasonflags( + backend, ext.only_some_reasons + ) + + if ext.full_name: + idp.distpoint = _encode_full_name(backend, ext.full_name) + + if ext.relative_name: + idp.distpoint = _encode_relative_name(backend, ext.relative_name) + + return idp + + +def _encode_crl_reason(backend, crl_reason): + asn1enum = backend._lib.ASN1_ENUMERATED_new() + backend.openssl_assert(asn1enum != backend._ffi.NULL) + asn1enum = backend._ffi.gc(asn1enum, backend._lib.ASN1_ENUMERATED_free) + res = backend._lib.ASN1_ENUMERATED_set( + asn1enum, _CRL_ENTRY_REASON_ENUM_TO_CODE[crl_reason.reason] + ) + backend.openssl_assert(res == 1) + + return asn1enum + + +def _encode_invalidity_date(backend, invalidity_date): + time = backend._lib.ASN1_GENERALIZEDTIME_set( + backend._ffi.NULL, + calendar.timegm(invalidity_date.invalidity_date.timetuple()), + ) + backend.openssl_assert(time != backend._ffi.NULL) + time = backend._ffi.gc(time, backend._lib.ASN1_GENERALIZEDTIME_free) + + return time + + +def _encode_certificate_policies(backend, certificate_policies): + cp = backend._lib.sk_POLICYINFO_new_null() + backend.openssl_assert(cp != backend._ffi.NULL) + cp = backend._ffi.gc(cp, backend._lib.sk_POLICYINFO_free) + for policy_info in certificate_policies: + pi = backend._lib.POLICYINFO_new() + backend.openssl_assert(pi != backend._ffi.NULL) + res = backend._lib.sk_POLICYINFO_push(cp, pi) + backend.openssl_assert(res >= 1) + oid = _txt2obj(backend, policy_info.policy_identifier.dotted_string) + pi.policyid = oid + if policy_info.policy_qualifiers: + pqis = backend._lib.sk_POLICYQUALINFO_new_null() + backend.openssl_assert(pqis != backend._ffi.NULL) + for qualifier in policy_info.policy_qualifiers: + pqi = backend._lib.POLICYQUALINFO_new() + backend.openssl_assert(pqi != backend._ffi.NULL) + res = backend._lib.sk_POLICYQUALINFO_push(pqis, pqi) + backend.openssl_assert(res >= 1) + if isinstance(qualifier, six.text_type): + pqi.pqualid = _txt2obj( + backend, x509.OID_CPS_QUALIFIER.dotted_string + ) + pqi.d.cpsuri = _encode_asn1_str( + backend, + qualifier.encode("ascii"), + ) + else: + assert isinstance(qualifier, x509.UserNotice) + pqi.pqualid = _txt2obj( + backend, x509.OID_CPS_USER_NOTICE.dotted_string + ) + un = backend._lib.USERNOTICE_new() + backend.openssl_assert(un != backend._ffi.NULL) + pqi.d.usernotice = un + if qualifier.explicit_text: + un.exptext = _encode_asn1_utf8_str( + backend, qualifier.explicit_text + ) + + un.noticeref = _encode_notice_reference( + backend, qualifier.notice_reference + ) + + pi.qualifiers = pqis + + return cp + + +def _encode_notice_reference(backend, notice): + if notice is None: + return backend._ffi.NULL + else: + nr = backend._lib.NOTICEREF_new() + backend.openssl_assert(nr != backend._ffi.NULL) + # organization is a required field + nr.organization = _encode_asn1_utf8_str(backend, notice.organization) + + notice_stack = backend._lib.sk_ASN1_INTEGER_new_null() + nr.noticenos = notice_stack + for number in notice.notice_numbers: + num = _encode_asn1_int(backend, number) + res = backend._lib.sk_ASN1_INTEGER_push(notice_stack, num) + backend.openssl_assert(res >= 1) + + return nr + + +def _txt2obj(backend, name): + """ + Converts a Python string with an ASN.1 object ID in dotted form to a + ASN1_OBJECT. + """ + name = name.encode("ascii") + obj = backend._lib.OBJ_txt2obj(name, 1) + backend.openssl_assert(obj != backend._ffi.NULL) + return obj + + +def _txt2obj_gc(backend, name): + obj = _txt2obj(backend, name) + obj = backend._ffi.gc(obj, backend._lib.ASN1_OBJECT_free) + return obj + + +def _encode_ocsp_nocheck(backend, ext): + # Doesn't need to be GC'd + return backend._lib.ASN1_NULL_new() + + +def _encode_key_usage(backend, key_usage): + set_bit = backend._lib.ASN1_BIT_STRING_set_bit + ku = backend._lib.ASN1_BIT_STRING_new() + ku = backend._ffi.gc(ku, backend._lib.ASN1_BIT_STRING_free) + res = set_bit(ku, 0, key_usage.digital_signature) + backend.openssl_assert(res == 1) + res = set_bit(ku, 1, key_usage.content_commitment) + backend.openssl_assert(res == 1) + res = set_bit(ku, 2, key_usage.key_encipherment) + backend.openssl_assert(res == 1) + res = set_bit(ku, 3, key_usage.data_encipherment) + backend.openssl_assert(res == 1) + res = set_bit(ku, 4, key_usage.key_agreement) + backend.openssl_assert(res == 1) + res = set_bit(ku, 5, key_usage.key_cert_sign) + backend.openssl_assert(res == 1) + res = set_bit(ku, 6, key_usage.crl_sign) + backend.openssl_assert(res == 1) + if key_usage.key_agreement: + res = set_bit(ku, 7, key_usage.encipher_only) + backend.openssl_assert(res == 1) + res = set_bit(ku, 8, key_usage.decipher_only) + backend.openssl_assert(res == 1) + else: + res = set_bit(ku, 7, 0) + backend.openssl_assert(res == 1) + res = set_bit(ku, 8, 0) + backend.openssl_assert(res == 1) + + return ku + + +def _encode_authority_key_identifier(backend, authority_keyid): + akid = backend._lib.AUTHORITY_KEYID_new() + backend.openssl_assert(akid != backend._ffi.NULL) + akid = backend._ffi.gc(akid, backend._lib.AUTHORITY_KEYID_free) + if authority_keyid.key_identifier is not None: + akid.keyid = _encode_asn1_str( + backend, + authority_keyid.key_identifier, + ) + + if authority_keyid.authority_cert_issuer is not None: + akid.issuer = _encode_general_names( + backend, authority_keyid.authority_cert_issuer + ) + + if authority_keyid.authority_cert_serial_number is not None: + akid.serial = _encode_asn1_int( + backend, authority_keyid.authority_cert_serial_number + ) + + return akid + + +def _encode_basic_constraints(backend, basic_constraints): + constraints = backend._lib.BASIC_CONSTRAINTS_new() + constraints = backend._ffi.gc( + constraints, backend._lib.BASIC_CONSTRAINTS_free + ) + constraints.ca = 255 if basic_constraints.ca else 0 + if basic_constraints.ca and basic_constraints.path_length is not None: + constraints.pathlen = _encode_asn1_int( + backend, basic_constraints.path_length + ) + + return constraints + + +def _encode_information_access(backend, info_access): + aia = backend._lib.sk_ACCESS_DESCRIPTION_new_null() + backend.openssl_assert(aia != backend._ffi.NULL) + aia = backend._ffi.gc( + aia, + lambda x: backend._lib.sk_ACCESS_DESCRIPTION_pop_free( + x, + backend._ffi.addressof( + backend._lib._original_lib, "ACCESS_DESCRIPTION_free" + ), + ), + ) + for access_description in info_access: + ad = backend._lib.ACCESS_DESCRIPTION_new() + method = _txt2obj( + backend, access_description.access_method.dotted_string + ) + _encode_general_name_preallocated( + backend, access_description.access_location, ad.location + ) + ad.method = method + res = backend._lib.sk_ACCESS_DESCRIPTION_push(aia, ad) + backend.openssl_assert(res >= 1) + + return aia + + +def _encode_general_names(backend, names): + general_names = backend._lib.GENERAL_NAMES_new() + backend.openssl_assert(general_names != backend._ffi.NULL) + for name in names: + gn = _encode_general_name(backend, name) + res = backend._lib.sk_GENERAL_NAME_push(general_names, gn) + backend.openssl_assert(res != 0) + + return general_names + + +def _encode_alt_name(backend, san): + general_names = _encode_general_names(backend, san) + general_names = backend._ffi.gc( + general_names, backend._lib.GENERAL_NAMES_free + ) + return general_names + + +def _encode_subject_key_identifier(backend, ski): + return _encode_asn1_str_gc(backend, ski.digest) + + +def _encode_general_name(backend, name): + gn = backend._lib.GENERAL_NAME_new() + _encode_general_name_preallocated(backend, name, gn) + return gn + + +def _encode_general_name_preallocated(backend, name, gn): + if isinstance(name, x509.DNSName): + backend.openssl_assert(gn != backend._ffi.NULL) + gn.type = backend._lib.GEN_DNS + + ia5 = backend._lib.ASN1_IA5STRING_new() + backend.openssl_assert(ia5 != backend._ffi.NULL) + # ia5strings are supposed to be ITU T.50 but to allow round-tripping + # of broken certs that encode utf8 we'll encode utf8 here too. + value = name.value.encode("utf8") + + res = backend._lib.ASN1_STRING_set(ia5, value, len(value)) + backend.openssl_assert(res == 1) + gn.d.dNSName = ia5 + elif isinstance(name, x509.RegisteredID): + backend.openssl_assert(gn != backend._ffi.NULL) + gn.type = backend._lib.GEN_RID + obj = backend._lib.OBJ_txt2obj( + name.value.dotted_string.encode("ascii"), 1 + ) + backend.openssl_assert(obj != backend._ffi.NULL) + gn.d.registeredID = obj + elif isinstance(name, x509.DirectoryName): + backend.openssl_assert(gn != backend._ffi.NULL) + dir_name = _encode_name(backend, name.value) + gn.type = backend._lib.GEN_DIRNAME + gn.d.directoryName = dir_name + elif isinstance(name, x509.IPAddress): + backend.openssl_assert(gn != backend._ffi.NULL) + if isinstance(name.value, ipaddress.IPv4Network): + packed = name.value.network_address.packed + utils.int_to_bytes( + ((1 << 32) - name.value.num_addresses), 4 + ) + elif isinstance(name.value, ipaddress.IPv6Network): + packed = name.value.network_address.packed + utils.int_to_bytes( + (1 << 128) - name.value.num_addresses, 16 + ) + else: + packed = name.value.packed + ipaddr = _encode_asn1_str(backend, packed) + gn.type = backend._lib.GEN_IPADD + gn.d.iPAddress = ipaddr + elif isinstance(name, x509.OtherName): + backend.openssl_assert(gn != backend._ffi.NULL) + other_name = backend._lib.OTHERNAME_new() + backend.openssl_assert(other_name != backend._ffi.NULL) + + type_id = backend._lib.OBJ_txt2obj( + name.type_id.dotted_string.encode("ascii"), 1 + ) + backend.openssl_assert(type_id != backend._ffi.NULL) + data = backend._ffi.new("unsigned char[]", name.value) + data_ptr_ptr = backend._ffi.new("unsigned char **") + data_ptr_ptr[0] = data + value = backend._lib.d2i_ASN1_TYPE( + backend._ffi.NULL, data_ptr_ptr, len(name.value) + ) + if value == backend._ffi.NULL: + backend._consume_errors() + raise ValueError("Invalid ASN.1 data") + other_name.type_id = type_id + other_name.value = value + gn.type = backend._lib.GEN_OTHERNAME + gn.d.otherName = other_name + elif isinstance(name, x509.RFC822Name): + backend.openssl_assert(gn != backend._ffi.NULL) + # ia5strings are supposed to be ITU T.50 but to allow round-tripping + # of broken certs that encode utf8 we'll encode utf8 here too. + data = name.value.encode("utf8") + asn1_str = _encode_asn1_str(backend, data) + gn.type = backend._lib.GEN_EMAIL + gn.d.rfc822Name = asn1_str + elif isinstance(name, x509.UniformResourceIdentifier): + backend.openssl_assert(gn != backend._ffi.NULL) + # ia5strings are supposed to be ITU T.50 but to allow round-tripping + # of broken certs that encode utf8 we'll encode utf8 here too. + data = name.value.encode("utf8") + asn1_str = _encode_asn1_str(backend, data) + gn.type = backend._lib.GEN_URI + gn.d.uniformResourceIdentifier = asn1_str + else: + raise ValueError("{} is an unknown GeneralName type".format(name)) + + +def _encode_extended_key_usage(backend, extended_key_usage): + eku = backend._lib.sk_ASN1_OBJECT_new_null() + eku = backend._ffi.gc(eku, backend._lib.sk_ASN1_OBJECT_free) + for oid in extended_key_usage: + obj = _txt2obj(backend, oid.dotted_string) + res = backend._lib.sk_ASN1_OBJECT_push(eku, obj) + backend.openssl_assert(res >= 1) + + return eku + + +_CRLREASONFLAGS = { + x509.ReasonFlags.key_compromise: 1, + x509.ReasonFlags.ca_compromise: 2, + x509.ReasonFlags.affiliation_changed: 3, + x509.ReasonFlags.superseded: 4, + x509.ReasonFlags.cessation_of_operation: 5, + x509.ReasonFlags.certificate_hold: 6, + x509.ReasonFlags.privilege_withdrawn: 7, + x509.ReasonFlags.aa_compromise: 8, +} + + +def _encode_reasonflags(backend, reasons): + bitmask = backend._lib.ASN1_BIT_STRING_new() + backend.openssl_assert(bitmask != backend._ffi.NULL) + for reason in reasons: + res = backend._lib.ASN1_BIT_STRING_set_bit( + bitmask, _CRLREASONFLAGS[reason], 1 + ) + backend.openssl_assert(res == 1) + + return bitmask + + +def _encode_full_name(backend, full_name): + dpn = backend._lib.DIST_POINT_NAME_new() + backend.openssl_assert(dpn != backend._ffi.NULL) + dpn.type = _DISTPOINT_TYPE_FULLNAME + dpn.name.fullname = _encode_general_names(backend, full_name) + return dpn + + +def _encode_relative_name(backend, relative_name): + dpn = backend._lib.DIST_POINT_NAME_new() + backend.openssl_assert(dpn != backend._ffi.NULL) + dpn.type = _DISTPOINT_TYPE_RELATIVENAME + dpn.name.relativename = _encode_sk_name_entry(backend, relative_name) + return dpn + + +def _encode_cdps_freshest_crl(backend, cdps): + cdp = backend._lib.sk_DIST_POINT_new_null() + cdp = backend._ffi.gc(cdp, backend._lib.sk_DIST_POINT_free) + for point in cdps: + dp = backend._lib.DIST_POINT_new() + backend.openssl_assert(dp != backend._ffi.NULL) + + if point.reasons: + dp.reasons = _encode_reasonflags(backend, point.reasons) + + if point.full_name: + dp.distpoint = _encode_full_name(backend, point.full_name) + + if point.relative_name: + dp.distpoint = _encode_relative_name(backend, point.relative_name) + + if point.crl_issuer: + dp.CRLissuer = _encode_general_names(backend, point.crl_issuer) + + res = backend._lib.sk_DIST_POINT_push(cdp, dp) + backend.openssl_assert(res >= 1) + + return cdp + + +def _encode_name_constraints(backend, name_constraints): + nc = backend._lib.NAME_CONSTRAINTS_new() + backend.openssl_assert(nc != backend._ffi.NULL) + nc = backend._ffi.gc(nc, backend._lib.NAME_CONSTRAINTS_free) + permitted = _encode_general_subtree( + backend, name_constraints.permitted_subtrees + ) + nc.permittedSubtrees = permitted + excluded = _encode_general_subtree( + backend, name_constraints.excluded_subtrees + ) + nc.excludedSubtrees = excluded + + return nc + + +def _encode_policy_constraints(backend, policy_constraints): + pc = backend._lib.POLICY_CONSTRAINTS_new() + backend.openssl_assert(pc != backend._ffi.NULL) + pc = backend._ffi.gc(pc, backend._lib.POLICY_CONSTRAINTS_free) + if policy_constraints.require_explicit_policy is not None: + pc.requireExplicitPolicy = _encode_asn1_int( + backend, policy_constraints.require_explicit_policy + ) + + if policy_constraints.inhibit_policy_mapping is not None: + pc.inhibitPolicyMapping = _encode_asn1_int( + backend, policy_constraints.inhibit_policy_mapping + ) + + return pc + + +def _encode_general_subtree(backend, subtrees): + if subtrees is None: + return backend._ffi.NULL + else: + general_subtrees = backend._lib.sk_GENERAL_SUBTREE_new_null() + for name in subtrees: + gs = backend._lib.GENERAL_SUBTREE_new() + gs.base = _encode_general_name(backend, name) + res = backend._lib.sk_GENERAL_SUBTREE_push(general_subtrees, gs) + backend.openssl_assert(res >= 1) + + return general_subtrees + + +def _encode_precert_signed_certificate_timestamps(backend, scts): + sct_stack = backend._lib.sk_SCT_new_null() + backend.openssl_assert(sct_stack != backend._ffi.NULL) + sct_stack = backend._ffi.gc(sct_stack, backend._lib.sk_SCT_free) + for sct in scts: + res = backend._lib.sk_SCT_push(sct_stack, sct._sct) + backend.openssl_assert(res >= 1) + return sct_stack + + +def _encode_nonce(backend, nonce): + return _encode_asn1_str_gc(backend, nonce.nonce) + + +_EXTENSION_ENCODE_HANDLERS = { + ExtensionOID.BASIC_CONSTRAINTS: _encode_basic_constraints, + ExtensionOID.SUBJECT_KEY_IDENTIFIER: _encode_subject_key_identifier, + ExtensionOID.KEY_USAGE: _encode_key_usage, + ExtensionOID.SUBJECT_ALTERNATIVE_NAME: _encode_alt_name, + ExtensionOID.ISSUER_ALTERNATIVE_NAME: _encode_alt_name, + ExtensionOID.EXTENDED_KEY_USAGE: _encode_extended_key_usage, + ExtensionOID.AUTHORITY_KEY_IDENTIFIER: _encode_authority_key_identifier, + ExtensionOID.CERTIFICATE_POLICIES: _encode_certificate_policies, + ExtensionOID.AUTHORITY_INFORMATION_ACCESS: _encode_information_access, + ExtensionOID.SUBJECT_INFORMATION_ACCESS: _encode_information_access, + ExtensionOID.CRL_DISTRIBUTION_POINTS: _encode_cdps_freshest_crl, + ExtensionOID.FRESHEST_CRL: _encode_cdps_freshest_crl, + ExtensionOID.INHIBIT_ANY_POLICY: _encode_inhibit_any_policy, + ExtensionOID.OCSP_NO_CHECK: _encode_ocsp_nocheck, + ExtensionOID.NAME_CONSTRAINTS: _encode_name_constraints, + ExtensionOID.POLICY_CONSTRAINTS: _encode_policy_constraints, + ExtensionOID.PRECERT_SIGNED_CERTIFICATE_TIMESTAMPS: ( + _encode_precert_signed_certificate_timestamps + ), +} + +_CRL_EXTENSION_ENCODE_HANDLERS = { + ExtensionOID.ISSUER_ALTERNATIVE_NAME: _encode_alt_name, + ExtensionOID.AUTHORITY_KEY_IDENTIFIER: _encode_authority_key_identifier, + ExtensionOID.AUTHORITY_INFORMATION_ACCESS: _encode_information_access, + ExtensionOID.CRL_NUMBER: _encode_crl_number_delta_crl_indicator, + ExtensionOID.DELTA_CRL_INDICATOR: _encode_crl_number_delta_crl_indicator, + ExtensionOID.ISSUING_DISTRIBUTION_POINT: _encode_issuing_dist_point, + ExtensionOID.FRESHEST_CRL: _encode_cdps_freshest_crl, +} + +_CRL_ENTRY_EXTENSION_ENCODE_HANDLERS = { + CRLEntryExtensionOID.CERTIFICATE_ISSUER: _encode_alt_name, + CRLEntryExtensionOID.CRL_REASON: _encode_crl_reason, + CRLEntryExtensionOID.INVALIDITY_DATE: _encode_invalidity_date, +} + +_OCSP_REQUEST_EXTENSION_ENCODE_HANDLERS = { + OCSPExtensionOID.NONCE: _encode_nonce, +} + +_OCSP_BASICRESP_EXTENSION_ENCODE_HANDLERS = { + OCSPExtensionOID.NONCE: _encode_nonce, +} diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/openssl/hashes.py b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/openssl/hashes.py new file mode 100644 index 0000000..764dce0 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/openssl/hashes.py @@ -0,0 +1,82 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function + + +from cryptography import utils +from cryptography.exceptions import UnsupportedAlgorithm, _Reasons +from cryptography.hazmat.primitives import hashes + + +@utils.register_interface(hashes.HashContext) +class _HashContext(object): + def __init__(self, backend, algorithm, ctx=None): + self._algorithm = algorithm + + self._backend = backend + + if ctx is None: + ctx = self._backend._lib.EVP_MD_CTX_new() + ctx = self._backend._ffi.gc( + ctx, self._backend._lib.EVP_MD_CTX_free + ) + evp_md = self._backend._evp_md_from_algorithm(algorithm) + if evp_md == self._backend._ffi.NULL: + raise UnsupportedAlgorithm( + "{} is not a supported hash on this backend.".format( + algorithm.name + ), + _Reasons.UNSUPPORTED_HASH, + ) + res = self._backend._lib.EVP_DigestInit_ex( + ctx, evp_md, self._backend._ffi.NULL + ) + self._backend.openssl_assert(res != 0) + + self._ctx = ctx + + algorithm = utils.read_only_property("_algorithm") + + def copy(self): + copied_ctx = self._backend._lib.EVP_MD_CTX_new() + copied_ctx = self._backend._ffi.gc( + copied_ctx, self._backend._lib.EVP_MD_CTX_free + ) + res = self._backend._lib.EVP_MD_CTX_copy_ex(copied_ctx, self._ctx) + self._backend.openssl_assert(res != 0) + return _HashContext(self._backend, self.algorithm, ctx=copied_ctx) + + def update(self, data): + data_ptr = self._backend._ffi.from_buffer(data) + res = self._backend._lib.EVP_DigestUpdate( + self._ctx, data_ptr, len(data) + ) + self._backend.openssl_assert(res != 0) + + def finalize(self): + if isinstance(self.algorithm, hashes.ExtendableOutputFunction): + # extendable output functions use a different finalize + return self._finalize_xof() + else: + buf = self._backend._ffi.new( + "unsigned char[]", self._backend._lib.EVP_MAX_MD_SIZE + ) + outlen = self._backend._ffi.new("unsigned int *") + res = self._backend._lib.EVP_DigestFinal_ex(self._ctx, buf, outlen) + self._backend.openssl_assert(res != 0) + self._backend.openssl_assert( + outlen[0] == self.algorithm.digest_size + ) + return self._backend._ffi.buffer(buf)[: outlen[0]] + + def _finalize_xof(self): + buf = self._backend._ffi.new( + "unsigned char[]", self.algorithm.digest_size + ) + res = self._backend._lib.EVP_DigestFinalXOF( + self._ctx, buf, self.algorithm.digest_size + ) + self._backend.openssl_assert(res != 0) + return self._backend._ffi.buffer(buf)[: self.algorithm.digest_size] diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/openssl/hmac.py b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/openssl/hmac.py new file mode 100644 index 0000000..1cc9d99 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/openssl/hmac.py @@ -0,0 +1,76 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function + + +from cryptography import utils +from cryptography.exceptions import ( + InvalidSignature, + UnsupportedAlgorithm, + _Reasons, +) +from cryptography.hazmat.primitives import constant_time, hashes + + +@utils.register_interface(hashes.HashContext) +class _HMACContext(object): + def __init__(self, backend, key, algorithm, ctx=None): + self._algorithm = algorithm + self._backend = backend + + if ctx is None: + ctx = self._backend._lib.HMAC_CTX_new() + self._backend.openssl_assert(ctx != self._backend._ffi.NULL) + ctx = self._backend._ffi.gc(ctx, self._backend._lib.HMAC_CTX_free) + evp_md = self._backend._evp_md_from_algorithm(algorithm) + if evp_md == self._backend._ffi.NULL: + raise UnsupportedAlgorithm( + "{} is not a supported hash on this backend".format( + algorithm.name + ), + _Reasons.UNSUPPORTED_HASH, + ) + key_ptr = self._backend._ffi.from_buffer(key) + res = self._backend._lib.HMAC_Init_ex( + ctx, key_ptr, len(key), evp_md, self._backend._ffi.NULL + ) + self._backend.openssl_assert(res != 0) + + self._ctx = ctx + self._key = key + + algorithm = utils.read_only_property("_algorithm") + + def copy(self): + copied_ctx = self._backend._lib.HMAC_CTX_new() + self._backend.openssl_assert(copied_ctx != self._backend._ffi.NULL) + copied_ctx = self._backend._ffi.gc( + copied_ctx, self._backend._lib.HMAC_CTX_free + ) + res = self._backend._lib.HMAC_CTX_copy(copied_ctx, self._ctx) + self._backend.openssl_assert(res != 0) + return _HMACContext( + self._backend, self._key, self.algorithm, ctx=copied_ctx + ) + + def update(self, data): + data_ptr = self._backend._ffi.from_buffer(data) + res = self._backend._lib.HMAC_Update(self._ctx, data_ptr, len(data)) + self._backend.openssl_assert(res != 0) + + def finalize(self): + buf = self._backend._ffi.new( + "unsigned char[]", self._backend._lib.EVP_MAX_MD_SIZE + ) + outlen = self._backend._ffi.new("unsigned int *") + res = self._backend._lib.HMAC_Final(self._ctx, buf, outlen) + self._backend.openssl_assert(res != 0) + self._backend.openssl_assert(outlen[0] == self.algorithm.digest_size) + return self._backend._ffi.buffer(buf)[: outlen[0]] + + def verify(self, signature): + digest = self.finalize() + if not constant_time.bytes_eq(digest, signature): + raise InvalidSignature("Signature did not match digest.") diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/openssl/ocsp.py b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/openssl/ocsp.py new file mode 100644 index 0000000..50c02e7 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/openssl/ocsp.py @@ -0,0 +1,401 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function + +import functools + +from cryptography import utils, x509 +from cryptography.exceptions import UnsupportedAlgorithm +from cryptography.hazmat.backends.openssl.decode_asn1 import ( + _CRL_ENTRY_REASON_CODE_TO_ENUM, + _asn1_integer_to_int, + _asn1_string_to_bytes, + _decode_x509_name, + _obj2txt, + _parse_asn1_generalized_time, +) +from cryptography.hazmat.backends.openssl.x509 import _Certificate +from cryptography.hazmat.primitives import serialization +from cryptography.x509.ocsp import ( + OCSPCertStatus, + OCSPRequest, + OCSPResponse, + OCSPResponseStatus, + _CERT_STATUS_TO_ENUM, + _OIDS_TO_HASH, + _RESPONSE_STATUS_TO_ENUM, +) + + +def _requires_successful_response(func): + @functools.wraps(func) + def wrapper(self, *args): + if self.response_status != OCSPResponseStatus.SUCCESSFUL: + raise ValueError( + "OCSP response status is not successful so the property " + "has no value" + ) + else: + return func(self, *args) + + return wrapper + + +def _issuer_key_hash(backend, cert_id): + key_hash = backend._ffi.new("ASN1_OCTET_STRING **") + res = backend._lib.OCSP_id_get0_info( + backend._ffi.NULL, + backend._ffi.NULL, + key_hash, + backend._ffi.NULL, + cert_id, + ) + backend.openssl_assert(res == 1) + backend.openssl_assert(key_hash[0] != backend._ffi.NULL) + return _asn1_string_to_bytes(backend, key_hash[0]) + + +def _issuer_name_hash(backend, cert_id): + name_hash = backend._ffi.new("ASN1_OCTET_STRING **") + res = backend._lib.OCSP_id_get0_info( + name_hash, + backend._ffi.NULL, + backend._ffi.NULL, + backend._ffi.NULL, + cert_id, + ) + backend.openssl_assert(res == 1) + backend.openssl_assert(name_hash[0] != backend._ffi.NULL) + return _asn1_string_to_bytes(backend, name_hash[0]) + + +def _serial_number(backend, cert_id): + num = backend._ffi.new("ASN1_INTEGER **") + res = backend._lib.OCSP_id_get0_info( + backend._ffi.NULL, backend._ffi.NULL, backend._ffi.NULL, num, cert_id + ) + backend.openssl_assert(res == 1) + backend.openssl_assert(num[0] != backend._ffi.NULL) + return _asn1_integer_to_int(backend, num[0]) + + +def _hash_algorithm(backend, cert_id): + asn1obj = backend._ffi.new("ASN1_OBJECT **") + res = backend._lib.OCSP_id_get0_info( + backend._ffi.NULL, + asn1obj, + backend._ffi.NULL, + backend._ffi.NULL, + cert_id, + ) + backend.openssl_assert(res == 1) + backend.openssl_assert(asn1obj[0] != backend._ffi.NULL) + oid = _obj2txt(backend, asn1obj[0]) + try: + return _OIDS_TO_HASH[oid] + except KeyError: + raise UnsupportedAlgorithm( + "Signature algorithm OID: {} not recognized".format(oid) + ) + + +@utils.register_interface(OCSPResponse) +class _OCSPResponse(object): + def __init__(self, backend, ocsp_response): + self._backend = backend + self._ocsp_response = ocsp_response + status = self._backend._lib.OCSP_response_status(self._ocsp_response) + self._backend.openssl_assert(status in _RESPONSE_STATUS_TO_ENUM) + self._status = _RESPONSE_STATUS_TO_ENUM[status] + if self._status is OCSPResponseStatus.SUCCESSFUL: + basic = self._backend._lib.OCSP_response_get1_basic( + self._ocsp_response + ) + self._backend.openssl_assert(basic != self._backend._ffi.NULL) + self._basic = self._backend._ffi.gc( + basic, self._backend._lib.OCSP_BASICRESP_free + ) + num_resp = self._backend._lib.OCSP_resp_count(self._basic) + if num_resp != 1: + raise ValueError( + "OCSP response contains more than one SINGLERESP structure" + ", which this library does not support. " + "{} found".format(num_resp) + ) + self._single = self._backend._lib.OCSP_resp_get0(self._basic, 0) + self._backend.openssl_assert( + self._single != self._backend._ffi.NULL + ) + self._cert_id = self._backend._lib.OCSP_SINGLERESP_get0_id( + self._single + ) + self._backend.openssl_assert( + self._cert_id != self._backend._ffi.NULL + ) + + response_status = utils.read_only_property("_status") + + @property + @_requires_successful_response + def signature_algorithm_oid(self): + alg = self._backend._lib.OCSP_resp_get0_tbs_sigalg(self._basic) + self._backend.openssl_assert(alg != self._backend._ffi.NULL) + oid = _obj2txt(self._backend, alg.algorithm) + return x509.ObjectIdentifier(oid) + + @property + @_requires_successful_response + def signature_hash_algorithm(self): + oid = self.signature_algorithm_oid + try: + return x509._SIG_OIDS_TO_HASH[oid] + except KeyError: + raise UnsupportedAlgorithm( + "Signature algorithm OID:{} not recognized".format(oid) + ) + + @property + @_requires_successful_response + def signature(self): + sig = self._backend._lib.OCSP_resp_get0_signature(self._basic) + self._backend.openssl_assert(sig != self._backend._ffi.NULL) + return _asn1_string_to_bytes(self._backend, sig) + + @property + @_requires_successful_response + def tbs_response_bytes(self): + respdata = self._backend._lib.OCSP_resp_get0_respdata(self._basic) + self._backend.openssl_assert(respdata != self._backend._ffi.NULL) + pp = self._backend._ffi.new("unsigned char **") + res = self._backend._lib.i2d_OCSP_RESPDATA(respdata, pp) + self._backend.openssl_assert(pp[0] != self._backend._ffi.NULL) + pp = self._backend._ffi.gc( + pp, lambda pointer: self._backend._lib.OPENSSL_free(pointer[0]) + ) + self._backend.openssl_assert(res > 0) + return self._backend._ffi.buffer(pp[0], res)[:] + + @property + @_requires_successful_response + def certificates(self): + sk_x509 = self._backend._lib.OCSP_resp_get0_certs(self._basic) + num = self._backend._lib.sk_X509_num(sk_x509) + certs = [] + for i in range(num): + x509 = self._backend._lib.sk_X509_value(sk_x509, i) + self._backend.openssl_assert(x509 != self._backend._ffi.NULL) + cert = _Certificate(self._backend, x509) + # We need to keep the OCSP response that the certificate came from + # alive until the Certificate object itself goes out of scope, so + # we give it a private reference. + cert._ocsp_resp = self + certs.append(cert) + + return certs + + @property + @_requires_successful_response + def responder_key_hash(self): + _, asn1_string = self._responder_key_name() + if asn1_string == self._backend._ffi.NULL: + return None + else: + return _asn1_string_to_bytes(self._backend, asn1_string) + + @property + @_requires_successful_response + def responder_name(self): + x509_name, _ = self._responder_key_name() + if x509_name == self._backend._ffi.NULL: + return None + else: + return _decode_x509_name(self._backend, x509_name) + + def _responder_key_name(self): + asn1_string = self._backend._ffi.new("ASN1_OCTET_STRING **") + x509_name = self._backend._ffi.new("X509_NAME **") + res = self._backend._lib.OCSP_resp_get0_id( + self._basic, asn1_string, x509_name + ) + self._backend.openssl_assert(res == 1) + return x509_name[0], asn1_string[0] + + @property + @_requires_successful_response + def produced_at(self): + produced_at = self._backend._lib.OCSP_resp_get0_produced_at( + self._basic + ) + return _parse_asn1_generalized_time(self._backend, produced_at) + + @property + @_requires_successful_response + def certificate_status(self): + status = self._backend._lib.OCSP_single_get0_status( + self._single, + self._backend._ffi.NULL, + self._backend._ffi.NULL, + self._backend._ffi.NULL, + self._backend._ffi.NULL, + ) + self._backend.openssl_assert(status in _CERT_STATUS_TO_ENUM) + return _CERT_STATUS_TO_ENUM[status] + + @property + @_requires_successful_response + def revocation_time(self): + if self.certificate_status is not OCSPCertStatus.REVOKED: + return None + + asn1_time = self._backend._ffi.new("ASN1_GENERALIZEDTIME **") + self._backend._lib.OCSP_single_get0_status( + self._single, + self._backend._ffi.NULL, + asn1_time, + self._backend._ffi.NULL, + self._backend._ffi.NULL, + ) + self._backend.openssl_assert(asn1_time[0] != self._backend._ffi.NULL) + return _parse_asn1_generalized_time(self._backend, asn1_time[0]) + + @property + @_requires_successful_response + def revocation_reason(self): + if self.certificate_status is not OCSPCertStatus.REVOKED: + return None + + reason_ptr = self._backend._ffi.new("int *") + self._backend._lib.OCSP_single_get0_status( + self._single, + reason_ptr, + self._backend._ffi.NULL, + self._backend._ffi.NULL, + self._backend._ffi.NULL, + ) + # If no reason is encoded OpenSSL returns -1 + if reason_ptr[0] == -1: + return None + else: + self._backend.openssl_assert( + reason_ptr[0] in _CRL_ENTRY_REASON_CODE_TO_ENUM + ) + return _CRL_ENTRY_REASON_CODE_TO_ENUM[reason_ptr[0]] + + @property + @_requires_successful_response + def this_update(self): + asn1_time = self._backend._ffi.new("ASN1_GENERALIZEDTIME **") + self._backend._lib.OCSP_single_get0_status( + self._single, + self._backend._ffi.NULL, + self._backend._ffi.NULL, + asn1_time, + self._backend._ffi.NULL, + ) + self._backend.openssl_assert(asn1_time[0] != self._backend._ffi.NULL) + return _parse_asn1_generalized_time(self._backend, asn1_time[0]) + + @property + @_requires_successful_response + def next_update(self): + asn1_time = self._backend._ffi.new("ASN1_GENERALIZEDTIME **") + self._backend._lib.OCSP_single_get0_status( + self._single, + self._backend._ffi.NULL, + self._backend._ffi.NULL, + self._backend._ffi.NULL, + asn1_time, + ) + if asn1_time[0] != self._backend._ffi.NULL: + return _parse_asn1_generalized_time(self._backend, asn1_time[0]) + else: + return None + + @property + @_requires_successful_response + def issuer_key_hash(self): + return _issuer_key_hash(self._backend, self._cert_id) + + @property + @_requires_successful_response + def issuer_name_hash(self): + return _issuer_name_hash(self._backend, self._cert_id) + + @property + @_requires_successful_response + def hash_algorithm(self): + return _hash_algorithm(self._backend, self._cert_id) + + @property + @_requires_successful_response + def serial_number(self): + return _serial_number(self._backend, self._cert_id) + + @utils.cached_property + @_requires_successful_response + def extensions(self): + return self._backend._ocsp_basicresp_ext_parser.parse(self._basic) + + @utils.cached_property + @_requires_successful_response + def single_extensions(self): + return self._backend._ocsp_singleresp_ext_parser.parse(self._single) + + def public_bytes(self, encoding): + if encoding is not serialization.Encoding.DER: + raise ValueError("The only allowed encoding value is Encoding.DER") + + bio = self._backend._create_mem_bio_gc() + res = self._backend._lib.i2d_OCSP_RESPONSE_bio( + bio, self._ocsp_response + ) + self._backend.openssl_assert(res > 0) + return self._backend._read_mem_bio(bio) + + +@utils.register_interface(OCSPRequest) +class _OCSPRequest(object): + def __init__(self, backend, ocsp_request): + if backend._lib.OCSP_request_onereq_count(ocsp_request) > 1: + raise NotImplementedError( + "OCSP request contains more than one request" + ) + self._backend = backend + self._ocsp_request = ocsp_request + self._request = self._backend._lib.OCSP_request_onereq_get0( + self._ocsp_request, 0 + ) + self._backend.openssl_assert(self._request != self._backend._ffi.NULL) + self._cert_id = self._backend._lib.OCSP_onereq_get0_id(self._request) + self._backend.openssl_assert(self._cert_id != self._backend._ffi.NULL) + + @property + def issuer_key_hash(self): + return _issuer_key_hash(self._backend, self._cert_id) + + @property + def issuer_name_hash(self): + return _issuer_name_hash(self._backend, self._cert_id) + + @property + def serial_number(self): + return _serial_number(self._backend, self._cert_id) + + @property + def hash_algorithm(self): + return _hash_algorithm(self._backend, self._cert_id) + + @utils.cached_property + def extensions(self): + return self._backend._ocsp_req_ext_parser.parse(self._ocsp_request) + + def public_bytes(self, encoding): + if encoding is not serialization.Encoding.DER: + raise ValueError("The only allowed encoding value is Encoding.DER") + + bio = self._backend._create_mem_bio_gc() + res = self._backend._lib.i2d_OCSP_REQUEST_bio(bio, self._ocsp_request) + self._backend.openssl_assert(res > 0) + return self._backend._read_mem_bio(bio) diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/openssl/poly1305.py b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/openssl/poly1305.py new file mode 100644 index 0000000..5699918 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/openssl/poly1305.py @@ -0,0 +1,65 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function + + +from cryptography.exceptions import InvalidSignature +from cryptography.hazmat.primitives import constant_time + + +_POLY1305_TAG_SIZE = 16 +_POLY1305_KEY_SIZE = 32 + + +class _Poly1305Context(object): + def __init__(self, backend, key): + self._backend = backend + + key_ptr = self._backend._ffi.from_buffer(key) + # This function copies the key into OpenSSL-owned memory so we don't + # need to retain it ourselves + evp_pkey = self._backend._lib.EVP_PKEY_new_raw_private_key( + self._backend._lib.NID_poly1305, + self._backend._ffi.NULL, + key_ptr, + len(key), + ) + self._backend.openssl_assert(evp_pkey != self._backend._ffi.NULL) + self._evp_pkey = self._backend._ffi.gc( + evp_pkey, self._backend._lib.EVP_PKEY_free + ) + ctx = self._backend._lib.EVP_MD_CTX_new() + self._backend.openssl_assert(ctx != self._backend._ffi.NULL) + self._ctx = self._backend._ffi.gc( + ctx, self._backend._lib.EVP_MD_CTX_free + ) + res = self._backend._lib.EVP_DigestSignInit( + self._ctx, + self._backend._ffi.NULL, + self._backend._ffi.NULL, + self._backend._ffi.NULL, + self._evp_pkey, + ) + self._backend.openssl_assert(res == 1) + + def update(self, data): + data_ptr = self._backend._ffi.from_buffer(data) + res = self._backend._lib.EVP_DigestSignUpdate( + self._ctx, data_ptr, len(data) + ) + self._backend.openssl_assert(res != 0) + + def finalize(self): + buf = self._backend._ffi.new("unsigned char[]", _POLY1305_TAG_SIZE) + outlen = self._backend._ffi.new("size_t *") + res = self._backend._lib.EVP_DigestSignFinal(self._ctx, buf, outlen) + self._backend.openssl_assert(res != 0) + self._backend.openssl_assert(outlen[0] == _POLY1305_TAG_SIZE) + return self._backend._ffi.buffer(buf)[: outlen[0]] + + def verify(self, tag): + mac = self.finalize() + if not constant_time.bytes_eq(mac, tag): + raise InvalidSignature("Value did not match computed tag.") diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/openssl/rsa.py b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/openssl/rsa.py new file mode 100644 index 0000000..82cd49c --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/openssl/rsa.py @@ -0,0 +1,516 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function + +from cryptography import utils +from cryptography.exceptions import ( + InvalidSignature, + UnsupportedAlgorithm, + _Reasons, +) +from cryptography.hazmat.backends.openssl.utils import ( + _calculate_digest_and_algorithm, + _check_not_prehashed, + _warn_sign_verify_deprecated, +) +from cryptography.hazmat.primitives import hashes +from cryptography.hazmat.primitives.asymmetric import ( + AsymmetricSignatureContext, + AsymmetricVerificationContext, + rsa, +) +from cryptography.hazmat.primitives.asymmetric.padding import ( + AsymmetricPadding, + MGF1, + OAEP, + PKCS1v15, + PSS, + calculate_max_pss_salt_length, +) +from cryptography.hazmat.primitives.asymmetric.rsa import ( + RSAPrivateKeyWithSerialization, + RSAPublicKeyWithSerialization, +) + + +def _get_rsa_pss_salt_length(pss, key, hash_algorithm): + salt = pss._salt_length + + if salt is MGF1.MAX_LENGTH or salt is PSS.MAX_LENGTH: + return calculate_max_pss_salt_length(key, hash_algorithm) + else: + return salt + + +def _enc_dec_rsa(backend, key, data, padding): + if not isinstance(padding, AsymmetricPadding): + raise TypeError("Padding must be an instance of AsymmetricPadding.") + + if isinstance(padding, PKCS1v15): + padding_enum = backend._lib.RSA_PKCS1_PADDING + elif isinstance(padding, OAEP): + padding_enum = backend._lib.RSA_PKCS1_OAEP_PADDING + + if not isinstance(padding._mgf, MGF1): + raise UnsupportedAlgorithm( + "Only MGF1 is supported by this backend.", + _Reasons.UNSUPPORTED_MGF, + ) + + if not backend.rsa_padding_supported(padding): + raise UnsupportedAlgorithm( + "This combination of padding and hash algorithm is not " + "supported by this backend.", + _Reasons.UNSUPPORTED_PADDING, + ) + + else: + raise UnsupportedAlgorithm( + "{} is not supported by this backend.".format(padding.name), + _Reasons.UNSUPPORTED_PADDING, + ) + + return _enc_dec_rsa_pkey_ctx(backend, key, data, padding_enum, padding) + + +def _enc_dec_rsa_pkey_ctx(backend, key, data, padding_enum, padding): + if isinstance(key, _RSAPublicKey): + init = backend._lib.EVP_PKEY_encrypt_init + crypt = backend._lib.EVP_PKEY_encrypt + else: + init = backend._lib.EVP_PKEY_decrypt_init + crypt = backend._lib.EVP_PKEY_decrypt + + pkey_ctx = backend._lib.EVP_PKEY_CTX_new(key._evp_pkey, backend._ffi.NULL) + backend.openssl_assert(pkey_ctx != backend._ffi.NULL) + pkey_ctx = backend._ffi.gc(pkey_ctx, backend._lib.EVP_PKEY_CTX_free) + res = init(pkey_ctx) + backend.openssl_assert(res == 1) + res = backend._lib.EVP_PKEY_CTX_set_rsa_padding(pkey_ctx, padding_enum) + backend.openssl_assert(res > 0) + buf_size = backend._lib.EVP_PKEY_size(key._evp_pkey) + backend.openssl_assert(buf_size > 0) + if isinstance(padding, OAEP) and backend._lib.Cryptography_HAS_RSA_OAEP_MD: + mgf1_md = backend._evp_md_non_null_from_algorithm( + padding._mgf._algorithm + ) + res = backend._lib.EVP_PKEY_CTX_set_rsa_mgf1_md(pkey_ctx, mgf1_md) + backend.openssl_assert(res > 0) + oaep_md = backend._evp_md_non_null_from_algorithm(padding._algorithm) + res = backend._lib.EVP_PKEY_CTX_set_rsa_oaep_md(pkey_ctx, oaep_md) + backend.openssl_assert(res > 0) + + if ( + isinstance(padding, OAEP) + and padding._label is not None + and len(padding._label) > 0 + ): + # set0_rsa_oaep_label takes ownership of the char * so we need to + # copy it into some new memory + labelptr = backend._lib.OPENSSL_malloc(len(padding._label)) + backend.openssl_assert(labelptr != backend._ffi.NULL) + backend._ffi.memmove(labelptr, padding._label, len(padding._label)) + res = backend._lib.EVP_PKEY_CTX_set0_rsa_oaep_label( + pkey_ctx, labelptr, len(padding._label) + ) + backend.openssl_assert(res == 1) + + outlen = backend._ffi.new("size_t *", buf_size) + buf = backend._ffi.new("unsigned char[]", buf_size) + # Everything from this line onwards is written with the goal of being as + # constant-time as is practical given the constraints of Python and our + # API. See Bleichenbacher's '98 attack on RSA, and its many many variants. + # As such, you should not attempt to change this (particularly to "clean it + # up") without understanding why it was written this way (see + # Chesterton's Fence), and without measuring to verify you have not + # introduced observable time differences. + res = crypt(pkey_ctx, buf, outlen, data, len(data)) + resbuf = backend._ffi.buffer(buf)[: outlen[0]] + backend._lib.ERR_clear_error() + if res <= 0: + raise ValueError("Encryption/decryption failed.") + return resbuf + + +def _rsa_sig_determine_padding(backend, key, padding, algorithm): + if not isinstance(padding, AsymmetricPadding): + raise TypeError("Expected provider of AsymmetricPadding.") + + pkey_size = backend._lib.EVP_PKEY_size(key._evp_pkey) + backend.openssl_assert(pkey_size > 0) + + if isinstance(padding, PKCS1v15): + # Hash algorithm is ignored for PKCS1v15-padding, may be None. + padding_enum = backend._lib.RSA_PKCS1_PADDING + elif isinstance(padding, PSS): + if not isinstance(padding._mgf, MGF1): + raise UnsupportedAlgorithm( + "Only MGF1 is supported by this backend.", + _Reasons.UNSUPPORTED_MGF, + ) + + # PSS padding requires a hash algorithm + if not isinstance(algorithm, hashes.HashAlgorithm): + raise TypeError("Expected instance of hashes.HashAlgorithm.") + + # Size of key in bytes - 2 is the maximum + # PSS signature length (salt length is checked later) + if pkey_size - algorithm.digest_size - 2 < 0: + raise ValueError( + "Digest too large for key size. Use a larger " + "key or different digest." + ) + + padding_enum = backend._lib.RSA_PKCS1_PSS_PADDING + else: + raise UnsupportedAlgorithm( + "{} is not supported by this backend.".format(padding.name), + _Reasons.UNSUPPORTED_PADDING, + ) + + return padding_enum + + +# Hash algorithm can be absent (None) to initialize the context without setting +# any message digest algorithm. This is currently only valid for the PKCS1v15 +# padding type, where it means that the signature data is encoded/decoded +# as provided, without being wrapped in a DigestInfo structure. +def _rsa_sig_setup(backend, padding, algorithm, key, init_func): + padding_enum = _rsa_sig_determine_padding(backend, key, padding, algorithm) + pkey_ctx = backend._lib.EVP_PKEY_CTX_new(key._evp_pkey, backend._ffi.NULL) + backend.openssl_assert(pkey_ctx != backend._ffi.NULL) + pkey_ctx = backend._ffi.gc(pkey_ctx, backend._lib.EVP_PKEY_CTX_free) + res = init_func(pkey_ctx) + backend.openssl_assert(res == 1) + if algorithm is not None: + evp_md = backend._evp_md_non_null_from_algorithm(algorithm) + res = backend._lib.EVP_PKEY_CTX_set_signature_md(pkey_ctx, evp_md) + if res == 0: + backend._consume_errors() + raise UnsupportedAlgorithm( + "{} is not supported by this backend for RSA signing.".format( + algorithm.name + ), + _Reasons.UNSUPPORTED_HASH, + ) + res = backend._lib.EVP_PKEY_CTX_set_rsa_padding(pkey_ctx, padding_enum) + if res <= 0: + backend._consume_errors() + raise UnsupportedAlgorithm( + "{} is not supported for the RSA signature operation.".format( + padding.name + ), + _Reasons.UNSUPPORTED_PADDING, + ) + if isinstance(padding, PSS): + res = backend._lib.EVP_PKEY_CTX_set_rsa_pss_saltlen( + pkey_ctx, _get_rsa_pss_salt_length(padding, key, algorithm) + ) + backend.openssl_assert(res > 0) + + mgf1_md = backend._evp_md_non_null_from_algorithm( + padding._mgf._algorithm + ) + res = backend._lib.EVP_PKEY_CTX_set_rsa_mgf1_md(pkey_ctx, mgf1_md) + backend.openssl_assert(res > 0) + + return pkey_ctx + + +def _rsa_sig_sign(backend, padding, algorithm, private_key, data): + pkey_ctx = _rsa_sig_setup( + backend, + padding, + algorithm, + private_key, + backend._lib.EVP_PKEY_sign_init, + ) + buflen = backend._ffi.new("size_t *") + res = backend._lib.EVP_PKEY_sign( + pkey_ctx, backend._ffi.NULL, buflen, data, len(data) + ) + backend.openssl_assert(res == 1) + buf = backend._ffi.new("unsigned char[]", buflen[0]) + res = backend._lib.EVP_PKEY_sign(pkey_ctx, buf, buflen, data, len(data)) + if res != 1: + errors = backend._consume_errors_with_text() + raise ValueError( + "Digest or salt length too long for key size. Use a larger key " + "or shorter salt length if you are specifying a PSS salt", + errors, + ) + + return backend._ffi.buffer(buf)[:] + + +def _rsa_sig_verify(backend, padding, algorithm, public_key, signature, data): + pkey_ctx = _rsa_sig_setup( + backend, + padding, + algorithm, + public_key, + backend._lib.EVP_PKEY_verify_init, + ) + res = backend._lib.EVP_PKEY_verify( + pkey_ctx, signature, len(signature), data, len(data) + ) + # The previous call can return negative numbers in the event of an + # error. This is not a signature failure but we need to fail if it + # occurs. + backend.openssl_assert(res >= 0) + if res == 0: + backend._consume_errors() + raise InvalidSignature + + +def _rsa_sig_recover(backend, padding, algorithm, public_key, signature): + pkey_ctx = _rsa_sig_setup( + backend, + padding, + algorithm, + public_key, + backend._lib.EVP_PKEY_verify_recover_init, + ) + + # Attempt to keep the rest of the code in this function as constant/time + # as possible. See the comment in _enc_dec_rsa_pkey_ctx. Note that the + # outlen parameter is used even though its value may be undefined in the + # error case. Due to the tolerant nature of Python slicing this does not + # trigger any exceptions. + maxlen = backend._lib.EVP_PKEY_size(public_key._evp_pkey) + backend.openssl_assert(maxlen > 0) + buf = backend._ffi.new("unsigned char[]", maxlen) + buflen = backend._ffi.new("size_t *", maxlen) + res = backend._lib.EVP_PKEY_verify_recover( + pkey_ctx, buf, buflen, signature, len(signature) + ) + resbuf = backend._ffi.buffer(buf)[: buflen[0]] + backend._lib.ERR_clear_error() + # Assume that all parameter errors are handled during the setup phase and + # any error here is due to invalid signature. + if res != 1: + raise InvalidSignature + return resbuf + + +@utils.register_interface(AsymmetricSignatureContext) +class _RSASignatureContext(object): + def __init__(self, backend, private_key, padding, algorithm): + self._backend = backend + self._private_key = private_key + + # We now call _rsa_sig_determine_padding in _rsa_sig_setup. However + # we need to make a pointless call to it here so we maintain the + # API of erroring on init with this context if the values are invalid. + _rsa_sig_determine_padding(backend, private_key, padding, algorithm) + self._padding = padding + self._algorithm = algorithm + self._hash_ctx = hashes.Hash(self._algorithm, self._backend) + + def update(self, data): + self._hash_ctx.update(data) + + def finalize(self): + return _rsa_sig_sign( + self._backend, + self._padding, + self._algorithm, + self._private_key, + self._hash_ctx.finalize(), + ) + + +@utils.register_interface(AsymmetricVerificationContext) +class _RSAVerificationContext(object): + def __init__(self, backend, public_key, signature, padding, algorithm): + self._backend = backend + self._public_key = public_key + self._signature = signature + self._padding = padding + # We now call _rsa_sig_determine_padding in _rsa_sig_setup. However + # we need to make a pointless call to it here so we maintain the + # API of erroring on init with this context if the values are invalid. + _rsa_sig_determine_padding(backend, public_key, padding, algorithm) + + padding = padding + self._algorithm = algorithm + self._hash_ctx = hashes.Hash(self._algorithm, self._backend) + + def update(self, data): + self._hash_ctx.update(data) + + def verify(self): + return _rsa_sig_verify( + self._backend, + self._padding, + self._algorithm, + self._public_key, + self._signature, + self._hash_ctx.finalize(), + ) + + +@utils.register_interface(RSAPrivateKeyWithSerialization) +class _RSAPrivateKey(object): + def __init__(self, backend, rsa_cdata, evp_pkey): + res = backend._lib.RSA_check_key(rsa_cdata) + if res != 1: + errors = backend._consume_errors_with_text() + raise ValueError("Invalid private key", errors) + + # Blinding is on by default in many versions of OpenSSL, but let's + # just be conservative here. + res = backend._lib.RSA_blinding_on(rsa_cdata, backend._ffi.NULL) + backend.openssl_assert(res == 1) + + self._backend = backend + self._rsa_cdata = rsa_cdata + self._evp_pkey = evp_pkey + + n = self._backend._ffi.new("BIGNUM **") + self._backend._lib.RSA_get0_key( + self._rsa_cdata, + n, + self._backend._ffi.NULL, + self._backend._ffi.NULL, + ) + self._backend.openssl_assert(n[0] != self._backend._ffi.NULL) + self._key_size = self._backend._lib.BN_num_bits(n[0]) + + key_size = utils.read_only_property("_key_size") + + def signer(self, padding, algorithm): + _warn_sign_verify_deprecated() + _check_not_prehashed(algorithm) + return _RSASignatureContext(self._backend, self, padding, algorithm) + + def decrypt(self, ciphertext, padding): + key_size_bytes = (self.key_size + 7) // 8 + if key_size_bytes != len(ciphertext): + raise ValueError("Ciphertext length must be equal to key size.") + + return _enc_dec_rsa(self._backend, self, ciphertext, padding) + + def public_key(self): + ctx = self._backend._lib.RSAPublicKey_dup(self._rsa_cdata) + self._backend.openssl_assert(ctx != self._backend._ffi.NULL) + ctx = self._backend._ffi.gc(ctx, self._backend._lib.RSA_free) + evp_pkey = self._backend._rsa_cdata_to_evp_pkey(ctx) + return _RSAPublicKey(self._backend, ctx, evp_pkey) + + def private_numbers(self): + n = self._backend._ffi.new("BIGNUM **") + e = self._backend._ffi.new("BIGNUM **") + d = self._backend._ffi.new("BIGNUM **") + p = self._backend._ffi.new("BIGNUM **") + q = self._backend._ffi.new("BIGNUM **") + dmp1 = self._backend._ffi.new("BIGNUM **") + dmq1 = self._backend._ffi.new("BIGNUM **") + iqmp = self._backend._ffi.new("BIGNUM **") + self._backend._lib.RSA_get0_key(self._rsa_cdata, n, e, d) + self._backend.openssl_assert(n[0] != self._backend._ffi.NULL) + self._backend.openssl_assert(e[0] != self._backend._ffi.NULL) + self._backend.openssl_assert(d[0] != self._backend._ffi.NULL) + self._backend._lib.RSA_get0_factors(self._rsa_cdata, p, q) + self._backend.openssl_assert(p[0] != self._backend._ffi.NULL) + self._backend.openssl_assert(q[0] != self._backend._ffi.NULL) + self._backend._lib.RSA_get0_crt_params( + self._rsa_cdata, dmp1, dmq1, iqmp + ) + self._backend.openssl_assert(dmp1[0] != self._backend._ffi.NULL) + self._backend.openssl_assert(dmq1[0] != self._backend._ffi.NULL) + self._backend.openssl_assert(iqmp[0] != self._backend._ffi.NULL) + return rsa.RSAPrivateNumbers( + p=self._backend._bn_to_int(p[0]), + q=self._backend._bn_to_int(q[0]), + d=self._backend._bn_to_int(d[0]), + dmp1=self._backend._bn_to_int(dmp1[0]), + dmq1=self._backend._bn_to_int(dmq1[0]), + iqmp=self._backend._bn_to_int(iqmp[0]), + public_numbers=rsa.RSAPublicNumbers( + e=self._backend._bn_to_int(e[0]), + n=self._backend._bn_to_int(n[0]), + ), + ) + + def private_bytes(self, encoding, format, encryption_algorithm): + return self._backend._private_key_bytes( + encoding, + format, + encryption_algorithm, + self, + self._evp_pkey, + self._rsa_cdata, + ) + + def sign(self, data, padding, algorithm): + data, algorithm = _calculate_digest_and_algorithm( + self._backend, data, algorithm + ) + return _rsa_sig_sign(self._backend, padding, algorithm, self, data) + + +@utils.register_interface(RSAPublicKeyWithSerialization) +class _RSAPublicKey(object): + def __init__(self, backend, rsa_cdata, evp_pkey): + self._backend = backend + self._rsa_cdata = rsa_cdata + self._evp_pkey = evp_pkey + + n = self._backend._ffi.new("BIGNUM **") + self._backend._lib.RSA_get0_key( + self._rsa_cdata, + n, + self._backend._ffi.NULL, + self._backend._ffi.NULL, + ) + self._backend.openssl_assert(n[0] != self._backend._ffi.NULL) + self._key_size = self._backend._lib.BN_num_bits(n[0]) + + key_size = utils.read_only_property("_key_size") + + def verifier(self, signature, padding, algorithm): + _warn_sign_verify_deprecated() + utils._check_bytes("signature", signature) + + _check_not_prehashed(algorithm) + return _RSAVerificationContext( + self._backend, self, signature, padding, algorithm + ) + + def encrypt(self, plaintext, padding): + return _enc_dec_rsa(self._backend, self, plaintext, padding) + + def public_numbers(self): + n = self._backend._ffi.new("BIGNUM **") + e = self._backend._ffi.new("BIGNUM **") + self._backend._lib.RSA_get0_key( + self._rsa_cdata, n, e, self._backend._ffi.NULL + ) + self._backend.openssl_assert(n[0] != self._backend._ffi.NULL) + self._backend.openssl_assert(e[0] != self._backend._ffi.NULL) + return rsa.RSAPublicNumbers( + e=self._backend._bn_to_int(e[0]), + n=self._backend._bn_to_int(n[0]), + ) + + def public_bytes(self, encoding, format): + return self._backend._public_key_bytes( + encoding, format, self, self._evp_pkey, self._rsa_cdata + ) + + def verify(self, signature, data, padding, algorithm): + data, algorithm = _calculate_digest_and_algorithm( + self._backend, data, algorithm + ) + return _rsa_sig_verify( + self._backend, padding, algorithm, self, signature, data + ) + + def recover_data_from_signature(self, signature, padding, algorithm): + _check_not_prehashed(algorithm) + return _rsa_sig_recover( + self._backend, padding, algorithm, self, signature + ) diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/openssl/utils.py b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/openssl/utils.py new file mode 100644 index 0000000..3d697d1 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/openssl/utils.py @@ -0,0 +1,66 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function + +import warnings + +from cryptography import utils +from cryptography.hazmat.primitives import hashes +from cryptography.hazmat.primitives.asymmetric.utils import Prehashed + + +def _evp_pkey_derive(backend, evp_pkey, peer_public_key): + ctx = backend._lib.EVP_PKEY_CTX_new(evp_pkey, backend._ffi.NULL) + backend.openssl_assert(ctx != backend._ffi.NULL) + ctx = backend._ffi.gc(ctx, backend._lib.EVP_PKEY_CTX_free) + res = backend._lib.EVP_PKEY_derive_init(ctx) + backend.openssl_assert(res == 1) + res = backend._lib.EVP_PKEY_derive_set_peer(ctx, peer_public_key._evp_pkey) + backend.openssl_assert(res == 1) + keylen = backend._ffi.new("size_t *") + res = backend._lib.EVP_PKEY_derive(ctx, backend._ffi.NULL, keylen) + backend.openssl_assert(res == 1) + backend.openssl_assert(keylen[0] > 0) + buf = backend._ffi.new("unsigned char[]", keylen[0]) + res = backend._lib.EVP_PKEY_derive(ctx, buf, keylen) + if res != 1: + raise ValueError("Null shared key derived from public/private pair.") + + return backend._ffi.buffer(buf, keylen[0])[:] + + +def _calculate_digest_and_algorithm(backend, data, algorithm): + if not isinstance(algorithm, Prehashed): + hash_ctx = hashes.Hash(algorithm, backend) + hash_ctx.update(data) + data = hash_ctx.finalize() + else: + algorithm = algorithm._algorithm + + if len(data) != algorithm.digest_size: + raise ValueError( + "The provided data must be the same length as the hash " + "algorithm's digest size." + ) + + return (data, algorithm) + + +def _check_not_prehashed(signature_algorithm): + if isinstance(signature_algorithm, Prehashed): + raise TypeError( + "Prehashed is only supported in the sign and verify methods. " + "It cannot be used with signer, verifier or " + "recover_data_from_signature." + ) + + +def _warn_sign_verify_deprecated(): + warnings.warn( + "signer and verifier have been deprecated. Please use sign " + "and verify instead.", + utils.PersistentlyDeprecated2017, + stacklevel=3, + ) diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/openssl/x25519.py b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/openssl/x25519.py new file mode 100644 index 0000000..4971c54 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/openssl/x25519.py @@ -0,0 +1,123 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function + +from cryptography import utils +from cryptography.hazmat.backends.openssl.utils import _evp_pkey_derive +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.asymmetric.x25519 import ( + X25519PrivateKey, + X25519PublicKey, +) + + +_X25519_KEY_SIZE = 32 + + +@utils.register_interface(X25519PublicKey) +class _X25519PublicKey(object): + def __init__(self, backend, evp_pkey): + self._backend = backend + self._evp_pkey = evp_pkey + + def public_bytes(self, encoding, format): + if ( + encoding is serialization.Encoding.Raw + or format is serialization.PublicFormat.Raw + ): + if ( + encoding is not serialization.Encoding.Raw + or format is not serialization.PublicFormat.Raw + ): + raise ValueError( + "When using Raw both encoding and format must be Raw" + ) + + return self._raw_public_bytes() + + return self._backend._public_key_bytes( + encoding, format, self, self._evp_pkey, None + ) + + def _raw_public_bytes(self): + ucharpp = self._backend._ffi.new("unsigned char **") + res = self._backend._lib.EVP_PKEY_get1_tls_encodedpoint( + self._evp_pkey, ucharpp + ) + self._backend.openssl_assert(res == 32) + self._backend.openssl_assert(ucharpp[0] != self._backend._ffi.NULL) + data = self._backend._ffi.gc( + ucharpp[0], self._backend._lib.OPENSSL_free + ) + return self._backend._ffi.buffer(data, res)[:] + + +@utils.register_interface(X25519PrivateKey) +class _X25519PrivateKey(object): + def __init__(self, backend, evp_pkey): + self._backend = backend + self._evp_pkey = evp_pkey + + def public_key(self): + bio = self._backend._create_mem_bio_gc() + res = self._backend._lib.i2d_PUBKEY_bio(bio, self._evp_pkey) + self._backend.openssl_assert(res == 1) + evp_pkey = self._backend._lib.d2i_PUBKEY_bio( + bio, self._backend._ffi.NULL + ) + self._backend.openssl_assert(evp_pkey != self._backend._ffi.NULL) + evp_pkey = self._backend._ffi.gc( + evp_pkey, self._backend._lib.EVP_PKEY_free + ) + return _X25519PublicKey(self._backend, evp_pkey) + + def exchange(self, peer_public_key): + if not isinstance(peer_public_key, X25519PublicKey): + raise TypeError("peer_public_key must be X25519PublicKey.") + + return _evp_pkey_derive(self._backend, self._evp_pkey, peer_public_key) + + def private_bytes(self, encoding, format, encryption_algorithm): + if ( + encoding is serialization.Encoding.Raw + or format is serialization.PublicFormat.Raw + ): + if ( + format is not serialization.PrivateFormat.Raw + or encoding is not serialization.Encoding.Raw + or not isinstance( + encryption_algorithm, serialization.NoEncryption + ) + ): + raise ValueError( + "When using Raw both encoding and format must be Raw " + "and encryption_algorithm must be NoEncryption()" + ) + + return self._raw_private_bytes() + + return self._backend._private_key_bytes( + encoding, format, encryption_algorithm, self, self._evp_pkey, None + ) + + def _raw_private_bytes(self): + # When we drop support for CRYPTOGRAPHY_OPENSSL_LESS_THAN_111 we can + # switch this to EVP_PKEY_new_raw_private_key + # The trick we use here is serializing to a PKCS8 key and just + # using the last 32 bytes, which is the key itself. + bio = self._backend._create_mem_bio_gc() + res = self._backend._lib.i2d_PKCS8PrivateKey_bio( + bio, + self._evp_pkey, + self._backend._ffi.NULL, + self._backend._ffi.NULL, + 0, + self._backend._ffi.NULL, + self._backend._ffi.NULL, + ) + self._backend.openssl_assert(res == 1) + pkcs8 = self._backend._read_mem_bio(bio) + self._backend.openssl_assert(len(pkcs8) == 48) + return pkcs8[-_X25519_KEY_SIZE:] diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/openssl/x448.py b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/openssl/x448.py new file mode 100644 index 0000000..7ebcdf8 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/openssl/x448.py @@ -0,0 +1,107 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function + +from cryptography import utils +from cryptography.hazmat.backends.openssl.utils import _evp_pkey_derive +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.asymmetric.x448 import ( + X448PrivateKey, + X448PublicKey, +) + +_X448_KEY_SIZE = 56 + + +@utils.register_interface(X448PublicKey) +class _X448PublicKey(object): + def __init__(self, backend, evp_pkey): + self._backend = backend + self._evp_pkey = evp_pkey + + def public_bytes(self, encoding, format): + if ( + encoding is serialization.Encoding.Raw + or format is serialization.PublicFormat.Raw + ): + if ( + encoding is not serialization.Encoding.Raw + or format is not serialization.PublicFormat.Raw + ): + raise ValueError( + "When using Raw both encoding and format must be Raw" + ) + + return self._raw_public_bytes() + + return self._backend._public_key_bytes( + encoding, format, self, self._evp_pkey, None + ) + + def _raw_public_bytes(self): + buf = self._backend._ffi.new("unsigned char []", _X448_KEY_SIZE) + buflen = self._backend._ffi.new("size_t *", _X448_KEY_SIZE) + res = self._backend._lib.EVP_PKEY_get_raw_public_key( + self._evp_pkey, buf, buflen + ) + self._backend.openssl_assert(res == 1) + self._backend.openssl_assert(buflen[0] == _X448_KEY_SIZE) + return self._backend._ffi.buffer(buf, _X448_KEY_SIZE)[:] + + +@utils.register_interface(X448PrivateKey) +class _X448PrivateKey(object): + def __init__(self, backend, evp_pkey): + self._backend = backend + self._evp_pkey = evp_pkey + + def public_key(self): + buf = self._backend._ffi.new("unsigned char []", _X448_KEY_SIZE) + buflen = self._backend._ffi.new("size_t *", _X448_KEY_SIZE) + res = self._backend._lib.EVP_PKEY_get_raw_public_key( + self._evp_pkey, buf, buflen + ) + self._backend.openssl_assert(res == 1) + self._backend.openssl_assert(buflen[0] == _X448_KEY_SIZE) + return self._backend.x448_load_public_bytes(buf) + + def exchange(self, peer_public_key): + if not isinstance(peer_public_key, X448PublicKey): + raise TypeError("peer_public_key must be X448PublicKey.") + + return _evp_pkey_derive(self._backend, self._evp_pkey, peer_public_key) + + def private_bytes(self, encoding, format, encryption_algorithm): + if ( + encoding is serialization.Encoding.Raw + or format is serialization.PublicFormat.Raw + ): + if ( + format is not serialization.PrivateFormat.Raw + or encoding is not serialization.Encoding.Raw + or not isinstance( + encryption_algorithm, serialization.NoEncryption + ) + ): + raise ValueError( + "When using Raw both encoding and format must be Raw " + "and encryption_algorithm must be NoEncryption()" + ) + + return self._raw_private_bytes() + + return self._backend._private_key_bytes( + encoding, format, encryption_algorithm, self, self._evp_pkey, None + ) + + def _raw_private_bytes(self): + buf = self._backend._ffi.new("unsigned char []", _X448_KEY_SIZE) + buflen = self._backend._ffi.new("size_t *", _X448_KEY_SIZE) + res = self._backend._lib.EVP_PKEY_get_raw_private_key( + self._evp_pkey, buf, buflen + ) + self._backend.openssl_assert(res == 1) + self._backend.openssl_assert(buflen[0] == _X448_KEY_SIZE) + return self._backend._ffi.buffer(buf, _X448_KEY_SIZE)[:] diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/openssl/x509.py b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/openssl/x509.py new file mode 100644 index 0000000..4d0dac7 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/backends/openssl/x509.py @@ -0,0 +1,587 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function + +import datetime +import operator + +from cryptography import utils, x509 +from cryptography.exceptions import UnsupportedAlgorithm +from cryptography.hazmat.backends.openssl.decode_asn1 import ( + _asn1_integer_to_int, + _asn1_string_to_bytes, + _decode_x509_name, + _obj2txt, + _parse_asn1_time, +) +from cryptography.hazmat.backends.openssl.encode_asn1 import ( + _encode_asn1_int_gc, + _txt2obj_gc, +) +from cryptography.hazmat.primitives import hashes, serialization +from cryptography.hazmat.primitives.asymmetric import dsa, ec, rsa +from cryptography.x509.name import _ASN1Type + + +@utils.register_interface(x509.Certificate) +class _Certificate(object): + def __init__(self, backend, x509_cert): + self._backend = backend + self._x509 = x509_cert + + version = self._backend._lib.X509_get_version(self._x509) + if version == 0: + self._version = x509.Version.v1 + elif version == 2: + self._version = x509.Version.v3 + else: + raise x509.InvalidVersion( + "{} is not a valid X509 version".format(version), version + ) + + def __repr__(self): + return "".format(self.subject) + + def __eq__(self, other): + if not isinstance(other, x509.Certificate): + return NotImplemented + + res = self._backend._lib.X509_cmp(self._x509, other._x509) + return res == 0 + + def __ne__(self, other): + return not self == other + + def __hash__(self): + return hash(self.public_bytes(serialization.Encoding.DER)) + + def __deepcopy__(self, memo): + return self + + def fingerprint(self, algorithm): + h = hashes.Hash(algorithm, self._backend) + h.update(self.public_bytes(serialization.Encoding.DER)) + return h.finalize() + + version = utils.read_only_property("_version") + + @property + def serial_number(self): + asn1_int = self._backend._lib.X509_get_serialNumber(self._x509) + self._backend.openssl_assert(asn1_int != self._backend._ffi.NULL) + return _asn1_integer_to_int(self._backend, asn1_int) + + def public_key(self): + pkey = self._backend._lib.X509_get_pubkey(self._x509) + if pkey == self._backend._ffi.NULL: + # Remove errors from the stack. + self._backend._consume_errors() + raise ValueError("Certificate public key is of an unknown type") + + pkey = self._backend._ffi.gc(pkey, self._backend._lib.EVP_PKEY_free) + + return self._backend._evp_pkey_to_public_key(pkey) + + @property + def not_valid_before(self): + asn1_time = self._backend._lib.X509_getm_notBefore(self._x509) + return _parse_asn1_time(self._backend, asn1_time) + + @property + def not_valid_after(self): + asn1_time = self._backend._lib.X509_getm_notAfter(self._x509) + return _parse_asn1_time(self._backend, asn1_time) + + @property + def issuer(self): + issuer = self._backend._lib.X509_get_issuer_name(self._x509) + self._backend.openssl_assert(issuer != self._backend._ffi.NULL) + return _decode_x509_name(self._backend, issuer) + + @property + def subject(self): + subject = self._backend._lib.X509_get_subject_name(self._x509) + self._backend.openssl_assert(subject != self._backend._ffi.NULL) + return _decode_x509_name(self._backend, subject) + + @property + def signature_hash_algorithm(self): + oid = self.signature_algorithm_oid + try: + return x509._SIG_OIDS_TO_HASH[oid] + except KeyError: + raise UnsupportedAlgorithm( + "Signature algorithm OID:{} not recognized".format(oid) + ) + + @property + def signature_algorithm_oid(self): + alg = self._backend._ffi.new("X509_ALGOR **") + self._backend._lib.X509_get0_signature( + self._backend._ffi.NULL, alg, self._x509 + ) + self._backend.openssl_assert(alg[0] != self._backend._ffi.NULL) + oid = _obj2txt(self._backend, alg[0].algorithm) + return x509.ObjectIdentifier(oid) + + @utils.cached_property + def extensions(self): + return self._backend._certificate_extension_parser.parse(self._x509) + + @property + def signature(self): + sig = self._backend._ffi.new("ASN1_BIT_STRING **") + self._backend._lib.X509_get0_signature( + sig, self._backend._ffi.NULL, self._x509 + ) + self._backend.openssl_assert(sig[0] != self._backend._ffi.NULL) + return _asn1_string_to_bytes(self._backend, sig[0]) + + @property + def tbs_certificate_bytes(self): + pp = self._backend._ffi.new("unsigned char **") + res = self._backend._lib.i2d_re_X509_tbs(self._x509, pp) + self._backend.openssl_assert(res > 0) + pp = self._backend._ffi.gc( + pp, lambda pointer: self._backend._lib.OPENSSL_free(pointer[0]) + ) + return self._backend._ffi.buffer(pp[0], res)[:] + + def public_bytes(self, encoding): + bio = self._backend._create_mem_bio_gc() + if encoding is serialization.Encoding.PEM: + res = self._backend._lib.PEM_write_bio_X509(bio, self._x509) + elif encoding is serialization.Encoding.DER: + res = self._backend._lib.i2d_X509_bio(bio, self._x509) + else: + raise TypeError("encoding must be an item from the Encoding enum") + + self._backend.openssl_assert(res == 1) + return self._backend._read_mem_bio(bio) + + +@utils.register_interface(x509.RevokedCertificate) +class _RevokedCertificate(object): + def __init__(self, backend, crl, x509_revoked): + self._backend = backend + # The X509_REVOKED_value is a X509_REVOKED * that has + # no reference counting. This means when X509_CRL_free is + # called then the CRL and all X509_REVOKED * are freed. Since + # you can retain a reference to a single revoked certificate + # and let the CRL fall out of scope we need to retain a + # private reference to the CRL inside the RevokedCertificate + # object to prevent the gc from being called inappropriately. + self._crl = crl + self._x509_revoked = x509_revoked + + @property + def serial_number(self): + asn1_int = self._backend._lib.X509_REVOKED_get0_serialNumber( + self._x509_revoked + ) + self._backend.openssl_assert(asn1_int != self._backend._ffi.NULL) + return _asn1_integer_to_int(self._backend, asn1_int) + + @property + def revocation_date(self): + return _parse_asn1_time( + self._backend, + self._backend._lib.X509_REVOKED_get0_revocationDate( + self._x509_revoked + ), + ) + + @utils.cached_property + def extensions(self): + return self._backend._revoked_cert_extension_parser.parse( + self._x509_revoked + ) + + +@utils.register_interface(x509.CertificateRevocationList) +class _CertificateRevocationList(object): + def __init__(self, backend, x509_crl): + self._backend = backend + self._x509_crl = x509_crl + + def __eq__(self, other): + if not isinstance(other, x509.CertificateRevocationList): + return NotImplemented + + res = self._backend._lib.X509_CRL_cmp(self._x509_crl, other._x509_crl) + return res == 0 + + def __ne__(self, other): + return not self == other + + def fingerprint(self, algorithm): + h = hashes.Hash(algorithm, self._backend) + bio = self._backend._create_mem_bio_gc() + res = self._backend._lib.i2d_X509_CRL_bio(bio, self._x509_crl) + self._backend.openssl_assert(res == 1) + der = self._backend._read_mem_bio(bio) + h.update(der) + return h.finalize() + + @utils.cached_property + def _sorted_crl(self): + # X509_CRL_get0_by_serial sorts in place, which breaks a variety of + # things we don't want to break (like iteration and the signature). + # Let's dupe it and sort that instead. + dup = self._backend._lib.X509_CRL_dup(self._x509_crl) + self._backend.openssl_assert(dup != self._backend._ffi.NULL) + dup = self._backend._ffi.gc(dup, self._backend._lib.X509_CRL_free) + return dup + + def get_revoked_certificate_by_serial_number(self, serial_number): + revoked = self._backend._ffi.new("X509_REVOKED **") + asn1_int = _encode_asn1_int_gc(self._backend, serial_number) + res = self._backend._lib.X509_CRL_get0_by_serial( + self._sorted_crl, revoked, asn1_int + ) + if res == 0: + return None + else: + self._backend.openssl_assert(revoked[0] != self._backend._ffi.NULL) + return _RevokedCertificate( + self._backend, self._sorted_crl, revoked[0] + ) + + @property + def signature_hash_algorithm(self): + oid = self.signature_algorithm_oid + try: + return x509._SIG_OIDS_TO_HASH[oid] + except KeyError: + raise UnsupportedAlgorithm( + "Signature algorithm OID:{} not recognized".format(oid) + ) + + @property + def signature_algorithm_oid(self): + alg = self._backend._ffi.new("X509_ALGOR **") + self._backend._lib.X509_CRL_get0_signature( + self._x509_crl, self._backend._ffi.NULL, alg + ) + self._backend.openssl_assert(alg[0] != self._backend._ffi.NULL) + oid = _obj2txt(self._backend, alg[0].algorithm) + return x509.ObjectIdentifier(oid) + + @property + def issuer(self): + issuer = self._backend._lib.X509_CRL_get_issuer(self._x509_crl) + self._backend.openssl_assert(issuer != self._backend._ffi.NULL) + return _decode_x509_name(self._backend, issuer) + + @property + def next_update(self): + nu = self._backend._lib.X509_CRL_get_nextUpdate(self._x509_crl) + self._backend.openssl_assert(nu != self._backend._ffi.NULL) + return _parse_asn1_time(self._backend, nu) + + @property + def last_update(self): + lu = self._backend._lib.X509_CRL_get_lastUpdate(self._x509_crl) + self._backend.openssl_assert(lu != self._backend._ffi.NULL) + return _parse_asn1_time(self._backend, lu) + + @property + def signature(self): + sig = self._backend._ffi.new("ASN1_BIT_STRING **") + self._backend._lib.X509_CRL_get0_signature( + self._x509_crl, sig, self._backend._ffi.NULL + ) + self._backend.openssl_assert(sig[0] != self._backend._ffi.NULL) + return _asn1_string_to_bytes(self._backend, sig[0]) + + @property + def tbs_certlist_bytes(self): + pp = self._backend._ffi.new("unsigned char **") + res = self._backend._lib.i2d_re_X509_CRL_tbs(self._x509_crl, pp) + self._backend.openssl_assert(res > 0) + pp = self._backend._ffi.gc( + pp, lambda pointer: self._backend._lib.OPENSSL_free(pointer[0]) + ) + return self._backend._ffi.buffer(pp[0], res)[:] + + def public_bytes(self, encoding): + bio = self._backend._create_mem_bio_gc() + if encoding is serialization.Encoding.PEM: + res = self._backend._lib.PEM_write_bio_X509_CRL( + bio, self._x509_crl + ) + elif encoding is serialization.Encoding.DER: + res = self._backend._lib.i2d_X509_CRL_bio(bio, self._x509_crl) + else: + raise TypeError("encoding must be an item from the Encoding enum") + + self._backend.openssl_assert(res == 1) + return self._backend._read_mem_bio(bio) + + def _revoked_cert(self, idx): + revoked = self._backend._lib.X509_CRL_get_REVOKED(self._x509_crl) + r = self._backend._lib.sk_X509_REVOKED_value(revoked, idx) + self._backend.openssl_assert(r != self._backend._ffi.NULL) + return _RevokedCertificate(self._backend, self, r) + + def __iter__(self): + for i in range(len(self)): + yield self._revoked_cert(i) + + def __getitem__(self, idx): + if isinstance(idx, slice): + start, stop, step = idx.indices(len(self)) + return [self._revoked_cert(i) for i in range(start, stop, step)] + else: + idx = operator.index(idx) + if idx < 0: + idx += len(self) + if not 0 <= idx < len(self): + raise IndexError + return self._revoked_cert(idx) + + def __len__(self): + revoked = self._backend._lib.X509_CRL_get_REVOKED(self._x509_crl) + if revoked == self._backend._ffi.NULL: + return 0 + else: + return self._backend._lib.sk_X509_REVOKED_num(revoked) + + @utils.cached_property + def extensions(self): + return self._backend._crl_extension_parser.parse(self._x509_crl) + + def is_signature_valid(self, public_key): + if not isinstance( + public_key, + (dsa.DSAPublicKey, rsa.RSAPublicKey, ec.EllipticCurvePublicKey), + ): + raise TypeError( + "Expecting one of DSAPublicKey, RSAPublicKey," + " or EllipticCurvePublicKey." + ) + res = self._backend._lib.X509_CRL_verify( + self._x509_crl, public_key._evp_pkey + ) + + if res != 1: + self._backend._consume_errors() + return False + + return True + + +@utils.register_interface(x509.CertificateSigningRequest) +class _CertificateSigningRequest(object): + def __init__(self, backend, x509_req): + self._backend = backend + self._x509_req = x509_req + + def __eq__(self, other): + if not isinstance(other, _CertificateSigningRequest): + return NotImplemented + + self_bytes = self.public_bytes(serialization.Encoding.DER) + other_bytes = other.public_bytes(serialization.Encoding.DER) + return self_bytes == other_bytes + + def __ne__(self, other): + return not self == other + + def __hash__(self): + return hash(self.public_bytes(serialization.Encoding.DER)) + + def public_key(self): + pkey = self._backend._lib.X509_REQ_get_pubkey(self._x509_req) + self._backend.openssl_assert(pkey != self._backend._ffi.NULL) + pkey = self._backend._ffi.gc(pkey, self._backend._lib.EVP_PKEY_free) + return self._backend._evp_pkey_to_public_key(pkey) + + @property + def subject(self): + subject = self._backend._lib.X509_REQ_get_subject_name(self._x509_req) + self._backend.openssl_assert(subject != self._backend._ffi.NULL) + return _decode_x509_name(self._backend, subject) + + @property + def signature_hash_algorithm(self): + oid = self.signature_algorithm_oid + try: + return x509._SIG_OIDS_TO_HASH[oid] + except KeyError: + raise UnsupportedAlgorithm( + "Signature algorithm OID:{} not recognized".format(oid) + ) + + @property + def signature_algorithm_oid(self): + alg = self._backend._ffi.new("X509_ALGOR **") + self._backend._lib.X509_REQ_get0_signature( + self._x509_req, self._backend._ffi.NULL, alg + ) + self._backend.openssl_assert(alg[0] != self._backend._ffi.NULL) + oid = _obj2txt(self._backend, alg[0].algorithm) + return x509.ObjectIdentifier(oid) + + @utils.cached_property + def extensions(self): + x509_exts = self._backend._lib.X509_REQ_get_extensions(self._x509_req) + x509_exts = self._backend._ffi.gc( + x509_exts, + lambda x: self._backend._lib.sk_X509_EXTENSION_pop_free( + x, + self._backend._ffi.addressof( + self._backend._lib._original_lib, "X509_EXTENSION_free" + ), + ), + ) + return self._backend._csr_extension_parser.parse(x509_exts) + + def public_bytes(self, encoding): + bio = self._backend._create_mem_bio_gc() + if encoding is serialization.Encoding.PEM: + res = self._backend._lib.PEM_write_bio_X509_REQ( + bio, self._x509_req + ) + elif encoding is serialization.Encoding.DER: + res = self._backend._lib.i2d_X509_REQ_bio(bio, self._x509_req) + else: + raise TypeError("encoding must be an item from the Encoding enum") + + self._backend.openssl_assert(res == 1) + return self._backend._read_mem_bio(bio) + + @property + def tbs_certrequest_bytes(self): + pp = self._backend._ffi.new("unsigned char **") + res = self._backend._lib.i2d_re_X509_REQ_tbs(self._x509_req, pp) + self._backend.openssl_assert(res > 0) + pp = self._backend._ffi.gc( + pp, lambda pointer: self._backend._lib.OPENSSL_free(pointer[0]) + ) + return self._backend._ffi.buffer(pp[0], res)[:] + + @property + def signature(self): + sig = self._backend._ffi.new("ASN1_BIT_STRING **") + self._backend._lib.X509_REQ_get0_signature( + self._x509_req, sig, self._backend._ffi.NULL + ) + self._backend.openssl_assert(sig[0] != self._backend._ffi.NULL) + return _asn1_string_to_bytes(self._backend, sig[0]) + + @property + def is_signature_valid(self): + pkey = self._backend._lib.X509_REQ_get_pubkey(self._x509_req) + self._backend.openssl_assert(pkey != self._backend._ffi.NULL) + pkey = self._backend._ffi.gc(pkey, self._backend._lib.EVP_PKEY_free) + res = self._backend._lib.X509_REQ_verify(self._x509_req, pkey) + + if res != 1: + self._backend._consume_errors() + return False + + return True + + def get_attribute_for_oid(self, oid): + obj = _txt2obj_gc(self._backend, oid.dotted_string) + pos = self._backend._lib.X509_REQ_get_attr_by_OBJ( + self._x509_req, obj, -1 + ) + if pos == -1: + raise x509.AttributeNotFound( + "No {} attribute was found".format(oid), oid + ) + + attr = self._backend._lib.X509_REQ_get_attr(self._x509_req, pos) + self._backend.openssl_assert(attr != self._backend._ffi.NULL) + # We don't support multiple valued attributes for now. + self._backend.openssl_assert( + self._backend._lib.X509_ATTRIBUTE_count(attr) == 1 + ) + asn1_type = self._backend._lib.X509_ATTRIBUTE_get0_type(attr, 0) + self._backend.openssl_assert(asn1_type != self._backend._ffi.NULL) + # We need this to ensure that our C type cast is safe. + # Also this should always be a sane string type, but we'll see if + # that is true in the real world... + if asn1_type.type not in ( + _ASN1Type.UTF8String.value, + _ASN1Type.PrintableString.value, + _ASN1Type.IA5String.value, + ): + raise ValueError( + "OID {} has a disallowed ASN.1 type: {}".format( + oid, asn1_type.type + ) + ) + + data = self._backend._lib.X509_ATTRIBUTE_get0_data( + attr, 0, asn1_type.type, self._backend._ffi.NULL + ) + self._backend.openssl_assert(data != self._backend._ffi.NULL) + # This cast is safe iff we assert on the type above to ensure + # that it is always a type of ASN1_STRING + data = self._backend._ffi.cast("ASN1_STRING *", data) + return _asn1_string_to_bytes(self._backend, data) + + +@utils.register_interface( + x509.certificate_transparency.SignedCertificateTimestamp +) +class _SignedCertificateTimestamp(object): + def __init__(self, backend, sct_list, sct): + self._backend = backend + # Keep the SCT_LIST that this SCT came from alive. + self._sct_list = sct_list + self._sct = sct + + @property + def version(self): + version = self._backend._lib.SCT_get_version(self._sct) + assert version == self._backend._lib.SCT_VERSION_V1 + return x509.certificate_transparency.Version.v1 + + @property + def log_id(self): + out = self._backend._ffi.new("unsigned char **") + log_id_length = self._backend._lib.SCT_get0_log_id(self._sct, out) + assert log_id_length >= 0 + return self._backend._ffi.buffer(out[0], log_id_length)[:] + + @property + def timestamp(self): + timestamp = self._backend._lib.SCT_get_timestamp(self._sct) + milliseconds = timestamp % 1000 + return datetime.datetime.utcfromtimestamp(timestamp // 1000).replace( + microsecond=milliseconds * 1000 + ) + + @property + def entry_type(self): + entry_type = self._backend._lib.SCT_get_log_entry_type(self._sct) + # We currently only support loading SCTs from the X.509 extension, so + # we only have precerts. + assert entry_type == self._backend._lib.CT_LOG_ENTRY_TYPE_PRECERT + return x509.certificate_transparency.LogEntryType.PRE_CERTIFICATE + + @property + def _signature(self): + ptrptr = self._backend._ffi.new("unsigned char **") + res = self._backend._lib.SCT_get0_signature(self._sct, ptrptr) + self._backend.openssl_assert(res > 0) + self._backend.openssl_assert(ptrptr[0] != self._backend._ffi.NULL) + return self._backend._ffi.buffer(ptrptr[0], res)[:] + + def __hash__(self): + return hash(self._signature) + + def __eq__(self, other): + if not isinstance(other, _SignedCertificateTimestamp): + return NotImplemented + + return self._signature == other._signature + + def __ne__(self, other): + return not self == other diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/bindings/__init__.py b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/bindings/__init__.py new file mode 100644 index 0000000..4b54088 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/bindings/__init__.py @@ -0,0 +1,5 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/bindings/__pycache__/__init__.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/bindings/__pycache__/__init__.cpython-39.pyc new file mode 100644 index 0000000..889781f Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/bindings/__pycache__/__init__.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/bindings/_openssl.pyd b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/bindings/_openssl.pyd new file mode 100644 index 0000000..8a6049c Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/bindings/_openssl.pyd differ diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/bindings/_padding.pyd b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/bindings/_padding.pyd new file mode 100644 index 0000000..c9eaf26 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/bindings/_padding.pyd differ diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/bindings/openssl/__init__.py b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/bindings/openssl/__init__.py new file mode 100644 index 0000000..4b54088 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/bindings/openssl/__init__.py @@ -0,0 +1,5 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/bindings/openssl/__pycache__/__init__.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/bindings/openssl/__pycache__/__init__.cpython-39.pyc new file mode 100644 index 0000000..5fd8f9f Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/bindings/openssl/__pycache__/__init__.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/bindings/openssl/__pycache__/_conditional.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/bindings/openssl/__pycache__/_conditional.cpython-39.pyc new file mode 100644 index 0000000..7e8a62d Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/bindings/openssl/__pycache__/_conditional.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/bindings/openssl/__pycache__/binding.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/bindings/openssl/__pycache__/binding.cpython-39.pyc new file mode 100644 index 0000000..09bf67e Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/bindings/openssl/__pycache__/binding.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/bindings/openssl/_conditional.py b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/bindings/openssl/_conditional.py new file mode 100644 index 0000000..ca50fed --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/bindings/openssl/_conditional.py @@ -0,0 +1,322 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function + + +def cryptography_has_ec2m(): + return [ + "EC_POINT_set_affine_coordinates_GF2m", + "EC_POINT_get_affine_coordinates_GF2m", + "EC_POINT_set_compressed_coordinates_GF2m", + ] + + +def cryptography_has_rsa_oaep_md(): + return [ + "EVP_PKEY_CTX_set_rsa_oaep_md", + ] + + +def cryptography_has_rsa_oaep_label(): + return [ + "EVP_PKEY_CTX_set0_rsa_oaep_label", + ] + + +def cryptography_has_ssl3_method(): + return [ + "SSLv3_method", + "SSLv3_client_method", + "SSLv3_server_method", + ] + + +def cryptography_has_102_verification(): + return [ + "X509_V_ERR_SUITE_B_INVALID_VERSION", + "X509_V_ERR_SUITE_B_INVALID_ALGORITHM", + "X509_V_ERR_SUITE_B_INVALID_CURVE", + "X509_V_ERR_SUITE_B_INVALID_SIGNATURE_ALGORITHM", + "X509_V_ERR_SUITE_B_LOS_NOT_ALLOWED", + "X509_V_ERR_SUITE_B_CANNOT_SIGN_P_384_WITH_P_256", + "X509_V_FLAG_SUITEB_128_LOS_ONLY", + "X509_V_FLAG_SUITEB_192_LOS", + "X509_V_FLAG_SUITEB_128_LOS", + ] + + +def cryptography_has_110_verification_params(): + return ["X509_CHECK_FLAG_NEVER_CHECK_SUBJECT"] + + +def cryptography_has_set_cert_cb(): + return [ + "SSL_CTX_set_cert_cb", + "SSL_set_cert_cb", + ] + + +def cryptography_has_ssl_st(): + return [ + "SSL_ST_BEFORE", + "SSL_ST_OK", + "SSL_ST_INIT", + "SSL_ST_RENEGOTIATE", + ] + + +def cryptography_has_tls_st(): + return [ + "TLS_ST_BEFORE", + "TLS_ST_OK", + ] + + +def cryptography_has_scrypt(): + return [ + "EVP_PBE_scrypt", + ] + + +def cryptography_has_evp_pkey_dhx(): + return [ + "EVP_PKEY_DHX", + ] + + +def cryptography_has_mem_functions(): + return [ + "Cryptography_CRYPTO_set_mem_functions", + ] + + +def cryptography_has_sct(): + return [ + "SCT_get_version", + "SCT_get_log_entry_type", + "SCT_get0_log_id", + "SCT_get0_signature", + "SCT_get_timestamp", + "SCT_set_source", + "sk_SCT_new_null", + "sk_SCT_free", + "sk_SCT_num", + "sk_SCT_value", + "sk_SCT_push", + "SCT_LIST_free", + "SCT_new", + "SCT_set1_log_id", + "SCT_set_timestamp", + "SCT_set_version", + "SCT_set_log_entry_type", + ] + + +def cryptography_has_x509_store_ctx_get_issuer(): + return [ + "X509_STORE_get_get_issuer", + "X509_STORE_set_get_issuer", + ] + + +def cryptography_has_ed448(): + return [ + "EVP_PKEY_ED448", + "NID_ED448", + ] + + +def cryptography_has_ed25519(): + return [ + "NID_ED25519", + "EVP_PKEY_ED25519", + ] + + +def cryptography_has_poly1305(): + return [ + "NID_poly1305", + "EVP_PKEY_POLY1305", + ] + + +def cryptography_has_oneshot_evp_digest_sign_verify(): + return [ + "EVP_DigestSign", + "EVP_DigestVerify", + ] + + +def cryptography_has_evp_digestfinal_xof(): + return [ + "EVP_DigestFinalXOF", + ] + + +def cryptography_has_evp_pkey_get_set_tls_encodedpoint(): + return [ + "EVP_PKEY_get1_tls_encodedpoint", + "EVP_PKEY_set1_tls_encodedpoint", + ] + + +def cryptography_has_fips(): + return [ + "FIPS_mode_set", + "FIPS_mode", + ] + + +def cryptography_has_ssl_sigalgs(): + return [ + "SSL_CTX_set1_sigalgs_list", + "SSL_get_sigalgs", + ] + + +def cryptography_has_psk(): + return [ + "SSL_CTX_use_psk_identity_hint", + "SSL_CTX_set_psk_server_callback", + "SSL_CTX_set_psk_client_callback", + ] + + +def cryptography_has_custom_ext(): + return [ + "SSL_CTX_add_client_custom_ext", + "SSL_CTX_add_server_custom_ext", + "SSL_extension_supported", + ] + + +def cryptography_has_openssl_cleanup(): + return [ + "OPENSSL_cleanup", + ] + + +def cryptography_has_tlsv13(): + return [ + "SSL_OP_NO_TLSv1_3", + "SSL_VERIFY_POST_HANDSHAKE", + "SSL_CTX_set_ciphersuites", + "SSL_verify_client_post_handshake", + "SSL_CTX_set_post_handshake_auth", + "SSL_set_post_handshake_auth", + "SSL_SESSION_get_max_early_data", + "SSL_write_early_data", + "SSL_read_early_data", + "SSL_CTX_set_max_early_data", + ] + + +def cryptography_has_keylog(): + return [ + "SSL_CTX_set_keylog_callback", + "SSL_CTX_get_keylog_callback", + ] + + +def cryptography_has_raw_key(): + return [ + "EVP_PKEY_new_raw_private_key", + "EVP_PKEY_new_raw_public_key", + "EVP_PKEY_get_raw_private_key", + "EVP_PKEY_get_raw_public_key", + ] + + +def cryptography_has_engine(): + return [ + "ENGINE_by_id", + "ENGINE_init", + "ENGINE_finish", + "ENGINE_get_default_RAND", + "ENGINE_set_default_RAND", + "ENGINE_unregister_RAND", + "ENGINE_ctrl_cmd", + "ENGINE_free", + "ENGINE_get_name", + "Cryptography_add_osrandom_engine", + "ENGINE_ctrl_cmd_string", + "ENGINE_load_builtin_engines", + "ENGINE_load_private_key", + "ENGINE_load_public_key", + ] + + +def cryptography_has_verified_chain(): + return [ + "SSL_get0_verified_chain", + ] + + +def cryptography_has_srtp(): + return [ + "SSL_CTX_set_tlsext_use_srtp", + "SSL_set_tlsext_use_srtp", + "SSL_get_selected_srtp_profile", + ] + + +def cryptography_has_get_proto_version(): + return [ + "SSL_CTX_get_min_proto_version", + "SSL_CTX_get_max_proto_version", + "SSL_get_min_proto_version", + "SSL_get_max_proto_version", + ] + + +# This is a mapping of +# {condition: function-returning-names-dependent-on-that-condition} so we can +# loop over them and delete unsupported names at runtime. It will be removed +# when cffi supports #if in cdef. We use functions instead of just a dict of +# lists so we can use coverage to measure which are used. +CONDITIONAL_NAMES = { + "Cryptography_HAS_EC2M": cryptography_has_ec2m, + "Cryptography_HAS_RSA_OAEP_MD": cryptography_has_rsa_oaep_md, + "Cryptography_HAS_RSA_OAEP_LABEL": cryptography_has_rsa_oaep_label, + "Cryptography_HAS_SSL3_METHOD": cryptography_has_ssl3_method, + "Cryptography_HAS_102_VERIFICATION": cryptography_has_102_verification, + "Cryptography_HAS_110_VERIFICATION_PARAMS": ( + cryptography_has_110_verification_params + ), + "Cryptography_HAS_SET_CERT_CB": cryptography_has_set_cert_cb, + "Cryptography_HAS_SSL_ST": cryptography_has_ssl_st, + "Cryptography_HAS_TLS_ST": cryptography_has_tls_st, + "Cryptography_HAS_SCRYPT": cryptography_has_scrypt, + "Cryptography_HAS_EVP_PKEY_DHX": cryptography_has_evp_pkey_dhx, + "Cryptography_HAS_MEM_FUNCTIONS": cryptography_has_mem_functions, + "Cryptography_HAS_SCT": cryptography_has_sct, + "Cryptography_HAS_X509_STORE_CTX_GET_ISSUER": ( + cryptography_has_x509_store_ctx_get_issuer + ), + "Cryptography_HAS_ED448": cryptography_has_ed448, + "Cryptography_HAS_ED25519": cryptography_has_ed25519, + "Cryptography_HAS_POLY1305": cryptography_has_poly1305, + "Cryptography_HAS_ONESHOT_EVP_DIGEST_SIGN_VERIFY": ( + cryptography_has_oneshot_evp_digest_sign_verify + ), + "Cryptography_HAS_EVP_PKEY_get_set_tls_encodedpoint": ( + cryptography_has_evp_pkey_get_set_tls_encodedpoint + ), + "Cryptography_HAS_FIPS": cryptography_has_fips, + "Cryptography_HAS_SIGALGS": cryptography_has_ssl_sigalgs, + "Cryptography_HAS_PSK": cryptography_has_psk, + "Cryptography_HAS_CUSTOM_EXT": cryptography_has_custom_ext, + "Cryptography_HAS_OPENSSL_CLEANUP": cryptography_has_openssl_cleanup, + "Cryptography_HAS_TLSv1_3": cryptography_has_tlsv13, + "Cryptography_HAS_KEYLOG": cryptography_has_keylog, + "Cryptography_HAS_RAW_KEY": cryptography_has_raw_key, + "Cryptography_HAS_EVP_DIGESTFINAL_XOF": ( + cryptography_has_evp_digestfinal_xof + ), + "Cryptography_HAS_ENGINE": cryptography_has_engine, + "Cryptography_HAS_VERIFIED_CHAIN": cryptography_has_verified_chain, + "Cryptography_HAS_SRTP": cryptography_has_srtp, + "Cryptography_HAS_GET_PROTO_VERSION": cryptography_has_get_proto_version, +} diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/bindings/openssl/binding.py b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/bindings/openssl/binding.py new file mode 100644 index 0000000..7a84a34 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/bindings/openssl/binding.py @@ -0,0 +1,172 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function + +import collections +import threading +import types + +import cryptography +from cryptography import utils +from cryptography.exceptions import InternalError +from cryptography.hazmat.bindings._openssl import ffi, lib +from cryptography.hazmat.bindings.openssl._conditional import CONDITIONAL_NAMES + +_OpenSSLErrorWithText = collections.namedtuple( + "_OpenSSLErrorWithText", ["code", "lib", "func", "reason", "reason_text"] +) + + +class _OpenSSLError(object): + def __init__(self, code, lib, func, reason): + self._code = code + self._lib = lib + self._func = func + self._reason = reason + + def _lib_reason_match(self, lib, reason): + return lib == self.lib and reason == self.reason + + code = utils.read_only_property("_code") + lib = utils.read_only_property("_lib") + func = utils.read_only_property("_func") + reason = utils.read_only_property("_reason") + + +def _consume_errors(lib): + errors = [] + while True: + code = lib.ERR_get_error() + if code == 0: + break + + err_lib = lib.ERR_GET_LIB(code) + err_func = lib.ERR_GET_FUNC(code) + err_reason = lib.ERR_GET_REASON(code) + + errors.append(_OpenSSLError(code, err_lib, err_func, err_reason)) + + return errors + + +def _errors_with_text(errors): + errors_with_text = [] + for err in errors: + buf = ffi.new("char[]", 256) + lib.ERR_error_string_n(err.code, buf, len(buf)) + err_text_reason = ffi.string(buf) + + errors_with_text.append( + _OpenSSLErrorWithText( + err.code, err.lib, err.func, err.reason, err_text_reason + ) + ) + + return errors_with_text + + +def _consume_errors_with_text(lib): + return _errors_with_text(_consume_errors(lib)) + + +def _openssl_assert(lib, ok, errors=None): + if not ok: + if errors is None: + errors = _consume_errors(lib) + errors_with_text = _errors_with_text(errors) + + raise InternalError( + "Unknown OpenSSL error. This error is commonly encountered when " + "another library is not cleaning up the OpenSSL error stack. If " + "you are using cryptography with another library that uses " + "OpenSSL try disabling it before reporting a bug. Otherwise " + "please file an issue at https://github.com/pyca/cryptography/" + "issues with information on how to reproduce " + "this. ({0!r})".format(errors_with_text), + errors_with_text, + ) + + +def build_conditional_library(lib, conditional_names): + conditional_lib = types.ModuleType("lib") + conditional_lib._original_lib = lib + excluded_names = set() + for condition, names_cb in conditional_names.items(): + if not getattr(lib, condition): + excluded_names.update(names_cb()) + + for attr in dir(lib): + if attr not in excluded_names: + setattr(conditional_lib, attr, getattr(lib, attr)) + + return conditional_lib + + +class Binding(object): + """ + OpenSSL API wrapper. + """ + + lib = None + ffi = ffi + _lib_loaded = False + _init_lock = threading.Lock() + + def __init__(self): + self._ensure_ffi_initialized() + + @classmethod + def _register_osrandom_engine(cls): + # Clear any errors extant in the queue before we start. In many + # scenarios other things may be interacting with OpenSSL in the same + # process space and it has proven untenable to assume that they will + # reliably clear the error queue. Once we clear it here we will + # error on any subsequent unexpected item in the stack. + cls.lib.ERR_clear_error() + if cls.lib.CRYPTOGRAPHY_NEEDS_OSRANDOM_ENGINE: + result = cls.lib.Cryptography_add_osrandom_engine() + _openssl_assert(cls.lib, result in (1, 2)) + + @classmethod + def _ensure_ffi_initialized(cls): + with cls._init_lock: + if not cls._lib_loaded: + cls.lib = build_conditional_library(lib, CONDITIONAL_NAMES) + cls._lib_loaded = True + # initialize the SSL library + cls.lib.SSL_library_init() + # adds all ciphers/digests for EVP + cls.lib.OpenSSL_add_all_algorithms() + cls._register_osrandom_engine() + + @classmethod + def init_static_locks(cls): + cls._ensure_ffi_initialized() + + +def _verify_package_version(version): + # Occasionally we run into situations where the version of the Python + # package does not match the version of the shared object that is loaded. + # This may occur in environments where multiple versions of cryptography + # are installed and available in the python path. To avoid errors cropping + # up later this code checks that the currently imported package and the + # shared object that were loaded have the same version and raise an + # ImportError if they do not + so_package_version = ffi.string(lib.CRYPTOGRAPHY_PACKAGE_VERSION) + if version.encode("ascii") != so_package_version: + raise ImportError( + "The version of cryptography does not match the loaded " + "shared object. This can happen if you have multiple copies of " + "cryptography installed in your Python path. Please try creating " + "a new virtual environment to resolve this issue. " + "Loaded python version: {}, shared object version: {}".format( + version, so_package_version + ) + ) + + +_verify_package_version(cryptography.__version__) + +Binding.init_static_locks() diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/__init__.py b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/__init__.py new file mode 100644 index 0000000..4b54088 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/__init__.py @@ -0,0 +1,5 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/__pycache__/__init__.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/__pycache__/__init__.cpython-39.pyc new file mode 100644 index 0000000..91b77d2 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/__pycache__/__init__.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/__pycache__/cmac.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/__pycache__/cmac.cpython-39.pyc new file mode 100644 index 0000000..28e36bd Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/__pycache__/cmac.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/__pycache__/constant_time.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/__pycache__/constant_time.cpython-39.pyc new file mode 100644 index 0000000..6f2d7f1 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/__pycache__/constant_time.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/__pycache__/hashes.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/__pycache__/hashes.cpython-39.pyc new file mode 100644 index 0000000..9aa412a Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/__pycache__/hashes.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/__pycache__/hmac.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/__pycache__/hmac.cpython-39.pyc new file mode 100644 index 0000000..8cb4276 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/__pycache__/hmac.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/__pycache__/keywrap.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/__pycache__/keywrap.cpython-39.pyc new file mode 100644 index 0000000..ce30188 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/__pycache__/keywrap.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/__pycache__/padding.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/__pycache__/padding.cpython-39.pyc new file mode 100644 index 0000000..009ef69 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/__pycache__/padding.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/__pycache__/poly1305.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/__pycache__/poly1305.cpython-39.pyc new file mode 100644 index 0000000..5082921 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/__pycache__/poly1305.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/asymmetric/__init__.py b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/asymmetric/__init__.py new file mode 100644 index 0000000..494a7a1 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/asymmetric/__init__.py @@ -0,0 +1,40 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function + +import abc + +import six + + +@six.add_metaclass(abc.ABCMeta) +class AsymmetricSignatureContext(object): + @abc.abstractmethod + def update(self, data): + """ + Processes the provided bytes and returns nothing. + """ + + @abc.abstractmethod + def finalize(self): + """ + Returns the signature as bytes. + """ + + +@six.add_metaclass(abc.ABCMeta) +class AsymmetricVerificationContext(object): + @abc.abstractmethod + def update(self, data): + """ + Processes the provided bytes and returns nothing. + """ + + @abc.abstractmethod + def verify(self): + """ + Raises an exception if the bytes provided to update do not match the + signature or the signature does not match the public key. + """ diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/asymmetric/__pycache__/__init__.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/asymmetric/__pycache__/__init__.cpython-39.pyc new file mode 100644 index 0000000..4cea8b4 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/asymmetric/__pycache__/__init__.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/asymmetric/__pycache__/dh.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/asymmetric/__pycache__/dh.cpython-39.pyc new file mode 100644 index 0000000..da57746 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/asymmetric/__pycache__/dh.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/asymmetric/__pycache__/dsa.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/asymmetric/__pycache__/dsa.cpython-39.pyc new file mode 100644 index 0000000..4b012d2 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/asymmetric/__pycache__/dsa.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/asymmetric/__pycache__/ec.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/asymmetric/__pycache__/ec.cpython-39.pyc new file mode 100644 index 0000000..ce085a1 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/asymmetric/__pycache__/ec.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/asymmetric/__pycache__/ed25519.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/asymmetric/__pycache__/ed25519.cpython-39.pyc new file mode 100644 index 0000000..7969462 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/asymmetric/__pycache__/ed25519.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/asymmetric/__pycache__/ed448.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/asymmetric/__pycache__/ed448.cpython-39.pyc new file mode 100644 index 0000000..3665f3c Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/asymmetric/__pycache__/ed448.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/asymmetric/__pycache__/padding.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/asymmetric/__pycache__/padding.cpython-39.pyc new file mode 100644 index 0000000..b8a2933 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/asymmetric/__pycache__/padding.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/asymmetric/__pycache__/rsa.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/asymmetric/__pycache__/rsa.cpython-39.pyc new file mode 100644 index 0000000..6046d67 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/asymmetric/__pycache__/rsa.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/asymmetric/__pycache__/utils.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/asymmetric/__pycache__/utils.cpython-39.pyc new file mode 100644 index 0000000..88acc3b Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/asymmetric/__pycache__/utils.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/asymmetric/__pycache__/x25519.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/asymmetric/__pycache__/x25519.cpython-39.pyc new file mode 100644 index 0000000..9bc031a Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/asymmetric/__pycache__/x25519.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/asymmetric/__pycache__/x448.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/asymmetric/__pycache__/x448.cpython-39.pyc new file mode 100644 index 0000000..70abe45 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/asymmetric/__pycache__/x448.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/asymmetric/dh.py b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/asymmetric/dh.py new file mode 100644 index 0000000..74a311d --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/asymmetric/dh.py @@ -0,0 +1,224 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function + +import abc + +import six + +from cryptography import utils +from cryptography.hazmat.backends import _get_backend + + +_MIN_MODULUS_SIZE = 512 + + +def generate_parameters(generator, key_size, backend=None): + backend = _get_backend(backend) + return backend.generate_dh_parameters(generator, key_size) + + +class DHPrivateNumbers(object): + def __init__(self, x, public_numbers): + if not isinstance(x, six.integer_types): + raise TypeError("x must be an integer.") + + if not isinstance(public_numbers, DHPublicNumbers): + raise TypeError( + "public_numbers must be an instance of " "DHPublicNumbers." + ) + + self._x = x + self._public_numbers = public_numbers + + def __eq__(self, other): + if not isinstance(other, DHPrivateNumbers): + return NotImplemented + + return ( + self._x == other._x + and self._public_numbers == other._public_numbers + ) + + def __ne__(self, other): + return not self == other + + def private_key(self, backend=None): + backend = _get_backend(backend) + return backend.load_dh_private_numbers(self) + + public_numbers = utils.read_only_property("_public_numbers") + x = utils.read_only_property("_x") + + +class DHPublicNumbers(object): + def __init__(self, y, parameter_numbers): + if not isinstance(y, six.integer_types): + raise TypeError("y must be an integer.") + + if not isinstance(parameter_numbers, DHParameterNumbers): + raise TypeError( + "parameters must be an instance of DHParameterNumbers." + ) + + self._y = y + self._parameter_numbers = parameter_numbers + + def __eq__(self, other): + if not isinstance(other, DHPublicNumbers): + return NotImplemented + + return ( + self._y == other._y + and self._parameter_numbers == other._parameter_numbers + ) + + def __ne__(self, other): + return not self == other + + def public_key(self, backend=None): + backend = _get_backend(backend) + return backend.load_dh_public_numbers(self) + + y = utils.read_only_property("_y") + parameter_numbers = utils.read_only_property("_parameter_numbers") + + +class DHParameterNumbers(object): + def __init__(self, p, g, q=None): + if not isinstance(p, six.integer_types) or not isinstance( + g, six.integer_types + ): + raise TypeError("p and g must be integers") + if q is not None and not isinstance(q, six.integer_types): + raise TypeError("q must be integer or None") + + if g < 2: + raise ValueError("DH generator must be 2 or greater") + + if p.bit_length() < _MIN_MODULUS_SIZE: + raise ValueError( + "p (modulus) must be at least {}-bit".format(_MIN_MODULUS_SIZE) + ) + + self._p = p + self._g = g + self._q = q + + def __eq__(self, other): + if not isinstance(other, DHParameterNumbers): + return NotImplemented + + return ( + self._p == other._p and self._g == other._g and self._q == other._q + ) + + def __ne__(self, other): + return not self == other + + def parameters(self, backend=None): + backend = _get_backend(backend) + return backend.load_dh_parameter_numbers(self) + + p = utils.read_only_property("_p") + g = utils.read_only_property("_g") + q = utils.read_only_property("_q") + + +@six.add_metaclass(abc.ABCMeta) +class DHParameters(object): + @abc.abstractmethod + def generate_private_key(self): + """ + Generates and returns a DHPrivateKey. + """ + + @abc.abstractmethod + def parameter_bytes(self, encoding, format): + """ + Returns the parameters serialized as bytes. + """ + + @abc.abstractmethod + def parameter_numbers(self): + """ + Returns a DHParameterNumbers. + """ + + +DHParametersWithSerialization = DHParameters + + +@six.add_metaclass(abc.ABCMeta) +class DHPrivateKey(object): + @abc.abstractproperty + def key_size(self): + """ + The bit length of the prime modulus. + """ + + @abc.abstractmethod + def public_key(self): + """ + The DHPublicKey associated with this private key. + """ + + @abc.abstractmethod + def parameters(self): + """ + The DHParameters object associated with this private key. + """ + + @abc.abstractmethod + def exchange(self, peer_public_key): + """ + Given peer's DHPublicKey, carry out the key exchange and + return shared key as bytes. + """ + + +@six.add_metaclass(abc.ABCMeta) +class DHPrivateKeyWithSerialization(DHPrivateKey): + @abc.abstractmethod + def private_numbers(self): + """ + Returns a DHPrivateNumbers. + """ + + @abc.abstractmethod + def private_bytes(self, encoding, format, encryption_algorithm): + """ + Returns the key serialized as bytes. + """ + + +@six.add_metaclass(abc.ABCMeta) +class DHPublicKey(object): + @abc.abstractproperty + def key_size(self): + """ + The bit length of the prime modulus. + """ + + @abc.abstractmethod + def parameters(self): + """ + The DHParameters object associated with this public key. + """ + + @abc.abstractmethod + def public_numbers(self): + """ + Returns a DHPublicNumbers. + """ + + @abc.abstractmethod + def public_bytes(self, encoding, format): + """ + Returns the key serialized as bytes. + """ + + +DHPublicKeyWithSerialization = DHPublicKey diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/asymmetric/dsa.py b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/asymmetric/dsa.py new file mode 100644 index 0000000..8ccc666 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/asymmetric/dsa.py @@ -0,0 +1,261 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function + +import abc + +import six + +from cryptography import utils +from cryptography.hazmat.backends import _get_backend + + +@six.add_metaclass(abc.ABCMeta) +class DSAParameters(object): + @abc.abstractmethod + def generate_private_key(self): + """ + Generates and returns a DSAPrivateKey. + """ + + +@six.add_metaclass(abc.ABCMeta) +class DSAParametersWithNumbers(DSAParameters): + @abc.abstractmethod + def parameter_numbers(self): + """ + Returns a DSAParameterNumbers. + """ + + +@six.add_metaclass(abc.ABCMeta) +class DSAPrivateKey(object): + @abc.abstractproperty + def key_size(self): + """ + The bit length of the prime modulus. + """ + + @abc.abstractmethod + def public_key(self): + """ + The DSAPublicKey associated with this private key. + """ + + @abc.abstractmethod + def parameters(self): + """ + The DSAParameters object associated with this private key. + """ + + @abc.abstractmethod + def signer(self, signature_algorithm): + """ + Returns an AsymmetricSignatureContext used for signing data. + """ + + @abc.abstractmethod + def sign(self, data, algorithm): + """ + Signs the data + """ + + +@six.add_metaclass(abc.ABCMeta) +class DSAPrivateKeyWithSerialization(DSAPrivateKey): + @abc.abstractmethod + def private_numbers(self): + """ + Returns a DSAPrivateNumbers. + """ + + @abc.abstractmethod + def private_bytes(self, encoding, format, encryption_algorithm): + """ + Returns the key serialized as bytes. + """ + + +@six.add_metaclass(abc.ABCMeta) +class DSAPublicKey(object): + @abc.abstractproperty + def key_size(self): + """ + The bit length of the prime modulus. + """ + + @abc.abstractmethod + def parameters(self): + """ + The DSAParameters object associated with this public key. + """ + + @abc.abstractmethod + def verifier(self, signature, signature_algorithm): + """ + Returns an AsymmetricVerificationContext used for signing data. + """ + + @abc.abstractmethod + def public_numbers(self): + """ + Returns a DSAPublicNumbers. + """ + + @abc.abstractmethod + def public_bytes(self, encoding, format): + """ + Returns the key serialized as bytes. + """ + + @abc.abstractmethod + def verify(self, signature, data, algorithm): + """ + Verifies the signature of the data. + """ + + +DSAPublicKeyWithSerialization = DSAPublicKey + + +def generate_parameters(key_size, backend=None): + backend = _get_backend(backend) + return backend.generate_dsa_parameters(key_size) + + +def generate_private_key(key_size, backend=None): + backend = _get_backend(backend) + return backend.generate_dsa_private_key_and_parameters(key_size) + + +def _check_dsa_parameters(parameters): + if parameters.p.bit_length() not in [1024, 2048, 3072, 4096]: + raise ValueError( + "p must be exactly 1024, 2048, 3072, or 4096 bits long" + ) + if parameters.q.bit_length() not in [160, 224, 256]: + raise ValueError("q must be exactly 160, 224, or 256 bits long") + + if not (1 < parameters.g < parameters.p): + raise ValueError("g, p don't satisfy 1 < g < p.") + + +def _check_dsa_private_numbers(numbers): + parameters = numbers.public_numbers.parameter_numbers + _check_dsa_parameters(parameters) + if numbers.x <= 0 or numbers.x >= parameters.q: + raise ValueError("x must be > 0 and < q.") + + if numbers.public_numbers.y != pow(parameters.g, numbers.x, parameters.p): + raise ValueError("y must be equal to (g ** x % p).") + + +class DSAParameterNumbers(object): + def __init__(self, p, q, g): + if ( + not isinstance(p, six.integer_types) + or not isinstance(q, six.integer_types) + or not isinstance(g, six.integer_types) + ): + raise TypeError( + "DSAParameterNumbers p, q, and g arguments must be integers." + ) + + self._p = p + self._q = q + self._g = g + + p = utils.read_only_property("_p") + q = utils.read_only_property("_q") + g = utils.read_only_property("_g") + + def parameters(self, backend=None): + backend = _get_backend(backend) + return backend.load_dsa_parameter_numbers(self) + + def __eq__(self, other): + if not isinstance(other, DSAParameterNumbers): + return NotImplemented + + return self.p == other.p and self.q == other.q and self.g == other.g + + def __ne__(self, other): + return not self == other + + def __repr__(self): + return ( + "".format(self=self) + ) + + +class DSAPublicNumbers(object): + def __init__(self, y, parameter_numbers): + if not isinstance(y, six.integer_types): + raise TypeError("DSAPublicNumbers y argument must be an integer.") + + if not isinstance(parameter_numbers, DSAParameterNumbers): + raise TypeError( + "parameter_numbers must be a DSAParameterNumbers instance." + ) + + self._y = y + self._parameter_numbers = parameter_numbers + + y = utils.read_only_property("_y") + parameter_numbers = utils.read_only_property("_parameter_numbers") + + def public_key(self, backend=None): + backend = _get_backend(backend) + return backend.load_dsa_public_numbers(self) + + def __eq__(self, other): + if not isinstance(other, DSAPublicNumbers): + return NotImplemented + + return ( + self.y == other.y + and self.parameter_numbers == other.parameter_numbers + ) + + def __ne__(self, other): + return not self == other + + def __repr__(self): + return ( + "".format(self=self) + ) + + +class DSAPrivateNumbers(object): + def __init__(self, x, public_numbers): + if not isinstance(x, six.integer_types): + raise TypeError("DSAPrivateNumbers x argument must be an integer.") + + if not isinstance(public_numbers, DSAPublicNumbers): + raise TypeError( + "public_numbers must be a DSAPublicNumbers instance." + ) + self._public_numbers = public_numbers + self._x = x + + x = utils.read_only_property("_x") + public_numbers = utils.read_only_property("_public_numbers") + + def private_key(self, backend=None): + backend = _get_backend(backend) + return backend.load_dsa_private_numbers(self) + + def __eq__(self, other): + if not isinstance(other, DSAPrivateNumbers): + return NotImplemented + + return ( + self.x == other.x and self.public_numbers == other.public_numbers + ) + + def __ne__(self, other): + return not self == other diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/asymmetric/ec.py b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/asymmetric/ec.py new file mode 100644 index 0000000..c7e694f --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/asymmetric/ec.py @@ -0,0 +1,502 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function + +import abc +import warnings + +import six + +from cryptography import utils +from cryptography.hazmat._oid import ObjectIdentifier +from cryptography.hazmat.backends import _get_backend + + +class EllipticCurveOID(object): + SECP192R1 = ObjectIdentifier("1.2.840.10045.3.1.1") + SECP224R1 = ObjectIdentifier("1.3.132.0.33") + SECP256K1 = ObjectIdentifier("1.3.132.0.10") + SECP256R1 = ObjectIdentifier("1.2.840.10045.3.1.7") + SECP384R1 = ObjectIdentifier("1.3.132.0.34") + SECP521R1 = ObjectIdentifier("1.3.132.0.35") + BRAINPOOLP256R1 = ObjectIdentifier("1.3.36.3.3.2.8.1.1.7") + BRAINPOOLP384R1 = ObjectIdentifier("1.3.36.3.3.2.8.1.1.11") + BRAINPOOLP512R1 = ObjectIdentifier("1.3.36.3.3.2.8.1.1.13") + SECT163K1 = ObjectIdentifier("1.3.132.0.1") + SECT163R2 = ObjectIdentifier("1.3.132.0.15") + SECT233K1 = ObjectIdentifier("1.3.132.0.26") + SECT233R1 = ObjectIdentifier("1.3.132.0.27") + SECT283K1 = ObjectIdentifier("1.3.132.0.16") + SECT283R1 = ObjectIdentifier("1.3.132.0.17") + SECT409K1 = ObjectIdentifier("1.3.132.0.36") + SECT409R1 = ObjectIdentifier("1.3.132.0.37") + SECT571K1 = ObjectIdentifier("1.3.132.0.38") + SECT571R1 = ObjectIdentifier("1.3.132.0.39") + + +@six.add_metaclass(abc.ABCMeta) +class EllipticCurve(object): + @abc.abstractproperty + def name(self): + """ + The name of the curve. e.g. secp256r1. + """ + + @abc.abstractproperty + def key_size(self): + """ + Bit size of a secret scalar for the curve. + """ + + +@six.add_metaclass(abc.ABCMeta) +class EllipticCurveSignatureAlgorithm(object): + @abc.abstractproperty + def algorithm(self): + """ + The digest algorithm used with this signature. + """ + + +@six.add_metaclass(abc.ABCMeta) +class EllipticCurvePrivateKey(object): + @abc.abstractmethod + def signer(self, signature_algorithm): + """ + Returns an AsymmetricSignatureContext used for signing data. + """ + + @abc.abstractmethod + def exchange(self, algorithm, peer_public_key): + """ + Performs a key exchange operation using the provided algorithm with the + provided peer's public key. + """ + + @abc.abstractmethod + def public_key(self): + """ + The EllipticCurvePublicKey for this private key. + """ + + @abc.abstractproperty + def curve(self): + """ + The EllipticCurve that this key is on. + """ + + @abc.abstractproperty + def key_size(self): + """ + Bit size of a secret scalar for the curve. + """ + + @abc.abstractmethod + def sign(self, data, signature_algorithm): + """ + Signs the data + """ + + +@six.add_metaclass(abc.ABCMeta) +class EllipticCurvePrivateKeyWithSerialization(EllipticCurvePrivateKey): + @abc.abstractmethod + def private_numbers(self): + """ + Returns an EllipticCurvePrivateNumbers. + """ + + @abc.abstractmethod + def private_bytes(self, encoding, format, encryption_algorithm): + """ + Returns the key serialized as bytes. + """ + + +@six.add_metaclass(abc.ABCMeta) +class EllipticCurvePublicKey(object): + @abc.abstractmethod + def verifier(self, signature, signature_algorithm): + """ + Returns an AsymmetricVerificationContext used for signing data. + """ + + @abc.abstractproperty + def curve(self): + """ + The EllipticCurve that this key is on. + """ + + @abc.abstractproperty + def key_size(self): + """ + Bit size of a secret scalar for the curve. + """ + + @abc.abstractmethod + def public_numbers(self): + """ + Returns an EllipticCurvePublicNumbers. + """ + + @abc.abstractmethod + def public_bytes(self, encoding, format): + """ + Returns the key serialized as bytes. + """ + + @abc.abstractmethod + def verify(self, signature, data, signature_algorithm): + """ + Verifies the signature of the data. + """ + + @classmethod + def from_encoded_point(cls, curve, data): + utils._check_bytes("data", data) + + if not isinstance(curve, EllipticCurve): + raise TypeError("curve must be an EllipticCurve instance") + + if len(data) == 0: + raise ValueError("data must not be an empty byte string") + + if six.indexbytes(data, 0) not in [0x02, 0x03, 0x04]: + raise ValueError("Unsupported elliptic curve point type") + + from cryptography.hazmat.backends.openssl.backend import backend + + return backend.load_elliptic_curve_public_bytes(curve, data) + + +EllipticCurvePublicKeyWithSerialization = EllipticCurvePublicKey + + +@utils.register_interface(EllipticCurve) +class SECT571R1(object): + name = "sect571r1" + key_size = 570 + + +@utils.register_interface(EllipticCurve) +class SECT409R1(object): + name = "sect409r1" + key_size = 409 + + +@utils.register_interface(EllipticCurve) +class SECT283R1(object): + name = "sect283r1" + key_size = 283 + + +@utils.register_interface(EllipticCurve) +class SECT233R1(object): + name = "sect233r1" + key_size = 233 + + +@utils.register_interface(EllipticCurve) +class SECT163R2(object): + name = "sect163r2" + key_size = 163 + + +@utils.register_interface(EllipticCurve) +class SECT571K1(object): + name = "sect571k1" + key_size = 571 + + +@utils.register_interface(EllipticCurve) +class SECT409K1(object): + name = "sect409k1" + key_size = 409 + + +@utils.register_interface(EllipticCurve) +class SECT283K1(object): + name = "sect283k1" + key_size = 283 + + +@utils.register_interface(EllipticCurve) +class SECT233K1(object): + name = "sect233k1" + key_size = 233 + + +@utils.register_interface(EllipticCurve) +class SECT163K1(object): + name = "sect163k1" + key_size = 163 + + +@utils.register_interface(EllipticCurve) +class SECP521R1(object): + name = "secp521r1" + key_size = 521 + + +@utils.register_interface(EllipticCurve) +class SECP384R1(object): + name = "secp384r1" + key_size = 384 + + +@utils.register_interface(EllipticCurve) +class SECP256R1(object): + name = "secp256r1" + key_size = 256 + + +@utils.register_interface(EllipticCurve) +class SECP256K1(object): + name = "secp256k1" + key_size = 256 + + +@utils.register_interface(EllipticCurve) +class SECP224R1(object): + name = "secp224r1" + key_size = 224 + + +@utils.register_interface(EllipticCurve) +class SECP192R1(object): + name = "secp192r1" + key_size = 192 + + +@utils.register_interface(EllipticCurve) +class BrainpoolP256R1(object): + name = "brainpoolP256r1" + key_size = 256 + + +@utils.register_interface(EllipticCurve) +class BrainpoolP384R1(object): + name = "brainpoolP384r1" + key_size = 384 + + +@utils.register_interface(EllipticCurve) +class BrainpoolP512R1(object): + name = "brainpoolP512r1" + key_size = 512 + + +_CURVE_TYPES = { + "prime192v1": SECP192R1, + "prime256v1": SECP256R1, + "secp192r1": SECP192R1, + "secp224r1": SECP224R1, + "secp256r1": SECP256R1, + "secp384r1": SECP384R1, + "secp521r1": SECP521R1, + "secp256k1": SECP256K1, + "sect163k1": SECT163K1, + "sect233k1": SECT233K1, + "sect283k1": SECT283K1, + "sect409k1": SECT409K1, + "sect571k1": SECT571K1, + "sect163r2": SECT163R2, + "sect233r1": SECT233R1, + "sect283r1": SECT283R1, + "sect409r1": SECT409R1, + "sect571r1": SECT571R1, + "brainpoolP256r1": BrainpoolP256R1, + "brainpoolP384r1": BrainpoolP384R1, + "brainpoolP512r1": BrainpoolP512R1, +} + + +@utils.register_interface(EllipticCurveSignatureAlgorithm) +class ECDSA(object): + def __init__(self, algorithm): + self._algorithm = algorithm + + algorithm = utils.read_only_property("_algorithm") + + +def generate_private_key(curve, backend=None): + backend = _get_backend(backend) + return backend.generate_elliptic_curve_private_key(curve) + + +def derive_private_key(private_value, curve, backend=None): + backend = _get_backend(backend) + if not isinstance(private_value, six.integer_types): + raise TypeError("private_value must be an integer type.") + + if private_value <= 0: + raise ValueError("private_value must be a positive integer.") + + if not isinstance(curve, EllipticCurve): + raise TypeError("curve must provide the EllipticCurve interface.") + + return backend.derive_elliptic_curve_private_key(private_value, curve) + + +class EllipticCurvePublicNumbers(object): + def __init__(self, x, y, curve): + if not isinstance(x, six.integer_types) or not isinstance( + y, six.integer_types + ): + raise TypeError("x and y must be integers.") + + if not isinstance(curve, EllipticCurve): + raise TypeError("curve must provide the EllipticCurve interface.") + + self._y = y + self._x = x + self._curve = curve + + def public_key(self, backend=None): + backend = _get_backend(backend) + return backend.load_elliptic_curve_public_numbers(self) + + def encode_point(self): + warnings.warn( + "encode_point has been deprecated on EllipticCurvePublicNumbers" + " and will be removed in a future version. Please use " + "EllipticCurvePublicKey.public_bytes to obtain both " + "compressed and uncompressed point encoding.", + utils.PersistentlyDeprecated2019, + stacklevel=2, + ) + # key_size is in bits. Convert to bytes and round up + byte_length = (self.curve.key_size + 7) // 8 + return ( + b"\x04" + + utils.int_to_bytes(self.x, byte_length) + + utils.int_to_bytes(self.y, byte_length) + ) + + @classmethod + def from_encoded_point(cls, curve, data): + if not isinstance(curve, EllipticCurve): + raise TypeError("curve must be an EllipticCurve instance") + + warnings.warn( + "Support for unsafe construction of public numbers from " + "encoded data will be removed in a future version. " + "Please use EllipticCurvePublicKey.from_encoded_point", + utils.PersistentlyDeprecated2019, + stacklevel=2, + ) + + if data.startswith(b"\x04"): + # key_size is in bits. Convert to bytes and round up + byte_length = (curve.key_size + 7) // 8 + if len(data) == 2 * byte_length + 1: + x = utils.int_from_bytes(data[1 : byte_length + 1], "big") + y = utils.int_from_bytes(data[byte_length + 1 :], "big") + return cls(x, y, curve) + else: + raise ValueError("Invalid elliptic curve point data length") + else: + raise ValueError("Unsupported elliptic curve point type") + + curve = utils.read_only_property("_curve") + x = utils.read_only_property("_x") + y = utils.read_only_property("_y") + + def __eq__(self, other): + if not isinstance(other, EllipticCurvePublicNumbers): + return NotImplemented + + return ( + self.x == other.x + and self.y == other.y + and self.curve.name == other.curve.name + and self.curve.key_size == other.curve.key_size + ) + + def __ne__(self, other): + return not self == other + + def __hash__(self): + return hash((self.x, self.y, self.curve.name, self.curve.key_size)) + + def __repr__(self): + return ( + "".format(self) + ) + + +class EllipticCurvePrivateNumbers(object): + def __init__(self, private_value, public_numbers): + if not isinstance(private_value, six.integer_types): + raise TypeError("private_value must be an integer.") + + if not isinstance(public_numbers, EllipticCurvePublicNumbers): + raise TypeError( + "public_numbers must be an EllipticCurvePublicNumbers " + "instance." + ) + + self._private_value = private_value + self._public_numbers = public_numbers + + def private_key(self, backend=None): + backend = _get_backend(backend) + return backend.load_elliptic_curve_private_numbers(self) + + private_value = utils.read_only_property("_private_value") + public_numbers = utils.read_only_property("_public_numbers") + + def __eq__(self, other): + if not isinstance(other, EllipticCurvePrivateNumbers): + return NotImplemented + + return ( + self.private_value == other.private_value + and self.public_numbers == other.public_numbers + ) + + def __ne__(self, other): + return not self == other + + def __hash__(self): + return hash((self.private_value, self.public_numbers)) + + +class ECDH(object): + pass + + +_OID_TO_CURVE = { + EllipticCurveOID.SECP192R1: SECP192R1, + EllipticCurveOID.SECP224R1: SECP224R1, + EllipticCurveOID.SECP256K1: SECP256K1, + EllipticCurveOID.SECP256R1: SECP256R1, + EllipticCurveOID.SECP384R1: SECP384R1, + EllipticCurveOID.SECP521R1: SECP521R1, + EllipticCurveOID.BRAINPOOLP256R1: BrainpoolP256R1, + EllipticCurveOID.BRAINPOOLP384R1: BrainpoolP384R1, + EllipticCurveOID.BRAINPOOLP512R1: BrainpoolP512R1, + EllipticCurveOID.SECT163K1: SECT163K1, + EllipticCurveOID.SECT163R2: SECT163R2, + EllipticCurveOID.SECT233K1: SECT233K1, + EllipticCurveOID.SECT233R1: SECT233R1, + EllipticCurveOID.SECT283K1: SECT283K1, + EllipticCurveOID.SECT283R1: SECT283R1, + EllipticCurveOID.SECT409K1: SECT409K1, + EllipticCurveOID.SECT409R1: SECT409R1, + EllipticCurveOID.SECT571K1: SECT571K1, + EllipticCurveOID.SECT571R1: SECT571R1, +} + + +def get_curve_for_oid(oid): + try: + return _OID_TO_CURVE[oid] + except KeyError: + raise LookupError( + "The provided object identifier has no matching elliptic " + "curve class" + ) diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/asymmetric/ed25519.py b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/asymmetric/ed25519.py new file mode 100644 index 0000000..2d07a02 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/asymmetric/ed25519.py @@ -0,0 +1,87 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function + +import abc + +import six + +from cryptography.exceptions import UnsupportedAlgorithm, _Reasons + + +_ED25519_KEY_SIZE = 32 +_ED25519_SIG_SIZE = 64 + + +@six.add_metaclass(abc.ABCMeta) +class Ed25519PublicKey(object): + @classmethod + def from_public_bytes(cls, data): + from cryptography.hazmat.backends.openssl.backend import backend + + if not backend.ed25519_supported(): + raise UnsupportedAlgorithm( + "ed25519 is not supported by this version of OpenSSL.", + _Reasons.UNSUPPORTED_PUBLIC_KEY_ALGORITHM, + ) + + return backend.ed25519_load_public_bytes(data) + + @abc.abstractmethod + def public_bytes(self, encoding, format): + """ + The serialized bytes of the public key. + """ + + @abc.abstractmethod + def verify(self, signature, data): + """ + Verify the signature. + """ + + +@six.add_metaclass(abc.ABCMeta) +class Ed25519PrivateKey(object): + @classmethod + def generate(cls): + from cryptography.hazmat.backends.openssl.backend import backend + + if not backend.ed25519_supported(): + raise UnsupportedAlgorithm( + "ed25519 is not supported by this version of OpenSSL.", + _Reasons.UNSUPPORTED_PUBLIC_KEY_ALGORITHM, + ) + + return backend.ed25519_generate_key() + + @classmethod + def from_private_bytes(cls, data): + from cryptography.hazmat.backends.openssl.backend import backend + + if not backend.ed25519_supported(): + raise UnsupportedAlgorithm( + "ed25519 is not supported by this version of OpenSSL.", + _Reasons.UNSUPPORTED_PUBLIC_KEY_ALGORITHM, + ) + + return backend.ed25519_load_private_bytes(data) + + @abc.abstractmethod + def public_key(self): + """ + The Ed25519PublicKey derived from the private key. + """ + + @abc.abstractmethod + def private_bytes(self, encoding, format, encryption_algorithm): + """ + The serialized bytes of the private key. + """ + + @abc.abstractmethod + def sign(self, data): + """ + Signs the data. + """ diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/asymmetric/ed448.py b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/asymmetric/ed448.py new file mode 100644 index 0000000..520ffcb --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/asymmetric/ed448.py @@ -0,0 +1,82 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function + +import abc + +import six + +from cryptography.exceptions import UnsupportedAlgorithm, _Reasons + + +@six.add_metaclass(abc.ABCMeta) +class Ed448PublicKey(object): + @classmethod + def from_public_bytes(cls, data): + from cryptography.hazmat.backends.openssl.backend import backend + + if not backend.ed448_supported(): + raise UnsupportedAlgorithm( + "ed448 is not supported by this version of OpenSSL.", + _Reasons.UNSUPPORTED_PUBLIC_KEY_ALGORITHM, + ) + + return backend.ed448_load_public_bytes(data) + + @abc.abstractmethod + def public_bytes(self, encoding, format): + """ + The serialized bytes of the public key. + """ + + @abc.abstractmethod + def verify(self, signature, data): + """ + Verify the signature. + """ + + +@six.add_metaclass(abc.ABCMeta) +class Ed448PrivateKey(object): + @classmethod + def generate(cls): + from cryptography.hazmat.backends.openssl.backend import backend + + if not backend.ed448_supported(): + raise UnsupportedAlgorithm( + "ed448 is not supported by this version of OpenSSL.", + _Reasons.UNSUPPORTED_PUBLIC_KEY_ALGORITHM, + ) + return backend.ed448_generate_key() + + @classmethod + def from_private_bytes(cls, data): + from cryptography.hazmat.backends.openssl.backend import backend + + if not backend.ed448_supported(): + raise UnsupportedAlgorithm( + "ed448 is not supported by this version of OpenSSL.", + _Reasons.UNSUPPORTED_PUBLIC_KEY_ALGORITHM, + ) + + return backend.ed448_load_private_bytes(data) + + @abc.abstractmethod + def public_key(self): + """ + The Ed448PublicKey derived from the private key. + """ + + @abc.abstractmethod + def sign(self, data): + """ + Signs the data. + """ + + @abc.abstractmethod + def private_bytes(self, encoding, format, encryption_algorithm): + """ + The serialized bytes of the private key. + """ diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/asymmetric/padding.py b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/asymmetric/padding.py new file mode 100644 index 0000000..fc8f6e2 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/asymmetric/padding.py @@ -0,0 +1,80 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function + +import abc + +import six + +from cryptography import utils +from cryptography.hazmat.primitives import hashes +from cryptography.hazmat.primitives.asymmetric import rsa + + +@six.add_metaclass(abc.ABCMeta) +class AsymmetricPadding(object): + @abc.abstractproperty + def name(self): + """ + A string naming this padding (e.g. "PSS", "PKCS1"). + """ + + +@utils.register_interface(AsymmetricPadding) +class PKCS1v15(object): + name = "EMSA-PKCS1-v1_5" + + +@utils.register_interface(AsymmetricPadding) +class PSS(object): + MAX_LENGTH = object() + name = "EMSA-PSS" + + def __init__(self, mgf, salt_length): + self._mgf = mgf + + if ( + not isinstance(salt_length, six.integer_types) + and salt_length is not self.MAX_LENGTH + ): + raise TypeError("salt_length must be an integer.") + + if salt_length is not self.MAX_LENGTH and salt_length < 0: + raise ValueError("salt_length must be zero or greater.") + + self._salt_length = salt_length + + +@utils.register_interface(AsymmetricPadding) +class OAEP(object): + name = "EME-OAEP" + + def __init__(self, mgf, algorithm, label): + if not isinstance(algorithm, hashes.HashAlgorithm): + raise TypeError("Expected instance of hashes.HashAlgorithm.") + + self._mgf = mgf + self._algorithm = algorithm + self._label = label + + +class MGF1(object): + MAX_LENGTH = object() + + def __init__(self, algorithm): + if not isinstance(algorithm, hashes.HashAlgorithm): + raise TypeError("Expected instance of hashes.HashAlgorithm.") + + self._algorithm = algorithm + + +def calculate_max_pss_salt_length(key, hash_algorithm): + if not isinstance(key, (rsa.RSAPrivateKey, rsa.RSAPublicKey)): + raise TypeError("key must be an RSA public or private key") + # bit length - 1 per RFC 3447 + emlen = (key.key_size + 6) // 8 + salt_length = emlen - hash_algorithm.digest_size - 2 + assert salt_length >= 0 + return salt_length diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/asymmetric/rsa.py b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/asymmetric/rsa.py new file mode 100644 index 0000000..ea16bbf --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/asymmetric/rsa.py @@ -0,0 +1,380 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function + +import abc + +try: + # Only available in math in 3.5+ + from math import gcd +except ImportError: + from fractions import gcd + +import six + +from cryptography import utils +from cryptography.exceptions import UnsupportedAlgorithm, _Reasons +from cryptography.hazmat.backends import _get_backend +from cryptography.hazmat.backends.interfaces import RSABackend + + +@six.add_metaclass(abc.ABCMeta) +class RSAPrivateKey(object): + @abc.abstractmethod + def signer(self, padding, algorithm): + """ + Returns an AsymmetricSignatureContext used for signing data. + """ + + @abc.abstractmethod + def decrypt(self, ciphertext, padding): + """ + Decrypts the provided ciphertext. + """ + + @abc.abstractproperty + def key_size(self): + """ + The bit length of the public modulus. + """ + + @abc.abstractmethod + def public_key(self): + """ + The RSAPublicKey associated with this private key. + """ + + @abc.abstractmethod + def sign(self, data, padding, algorithm): + """ + Signs the data. + """ + + +@six.add_metaclass(abc.ABCMeta) +class RSAPrivateKeyWithSerialization(RSAPrivateKey): + @abc.abstractmethod + def private_numbers(self): + """ + Returns an RSAPrivateNumbers. + """ + + @abc.abstractmethod + def private_bytes(self, encoding, format, encryption_algorithm): + """ + Returns the key serialized as bytes. + """ + + +@six.add_metaclass(abc.ABCMeta) +class RSAPublicKey(object): + @abc.abstractmethod + def verifier(self, signature, padding, algorithm): + """ + Returns an AsymmetricVerificationContext used for verifying signatures. + """ + + @abc.abstractmethod + def encrypt(self, plaintext, padding): + """ + Encrypts the given plaintext. + """ + + @abc.abstractproperty + def key_size(self): + """ + The bit length of the public modulus. + """ + + @abc.abstractmethod + def public_numbers(self): + """ + Returns an RSAPublicNumbers + """ + + @abc.abstractmethod + def public_bytes(self, encoding, format): + """ + Returns the key serialized as bytes. + """ + + @abc.abstractmethod + def verify(self, signature, data, padding, algorithm): + """ + Verifies the signature of the data. + """ + + @abc.abstractmethod + def recover_data_from_signature(self, signature, padding, algorithm): + """ + Recovers the original data from the signature. + """ + + +RSAPublicKeyWithSerialization = RSAPublicKey + + +def generate_private_key(public_exponent, key_size, backend=None): + backend = _get_backend(backend) + if not isinstance(backend, RSABackend): + raise UnsupportedAlgorithm( + "Backend object does not implement RSABackend.", + _Reasons.BACKEND_MISSING_INTERFACE, + ) + + _verify_rsa_parameters(public_exponent, key_size) + return backend.generate_rsa_private_key(public_exponent, key_size) + + +def _verify_rsa_parameters(public_exponent, key_size): + if public_exponent not in (3, 65537): + raise ValueError( + "public_exponent must be either 3 (for legacy compatibility) or " + "65537. Almost everyone should choose 65537 here!" + ) + + if key_size < 512: + raise ValueError("key_size must be at least 512-bits.") + + +def _check_private_key_components( + p, q, private_exponent, dmp1, dmq1, iqmp, public_exponent, modulus +): + if modulus < 3: + raise ValueError("modulus must be >= 3.") + + if p >= modulus: + raise ValueError("p must be < modulus.") + + if q >= modulus: + raise ValueError("q must be < modulus.") + + if dmp1 >= modulus: + raise ValueError("dmp1 must be < modulus.") + + if dmq1 >= modulus: + raise ValueError("dmq1 must be < modulus.") + + if iqmp >= modulus: + raise ValueError("iqmp must be < modulus.") + + if private_exponent >= modulus: + raise ValueError("private_exponent must be < modulus.") + + if public_exponent < 3 or public_exponent >= modulus: + raise ValueError("public_exponent must be >= 3 and < modulus.") + + if public_exponent & 1 == 0: + raise ValueError("public_exponent must be odd.") + + if dmp1 & 1 == 0: + raise ValueError("dmp1 must be odd.") + + if dmq1 & 1 == 0: + raise ValueError("dmq1 must be odd.") + + if p * q != modulus: + raise ValueError("p*q must equal modulus.") + + +def _check_public_key_components(e, n): + if n < 3: + raise ValueError("n must be >= 3.") + + if e < 3 or e >= n: + raise ValueError("e must be >= 3 and < n.") + + if e & 1 == 0: + raise ValueError("e must be odd.") + + +def _modinv(e, m): + """ + Modular Multiplicative Inverse. Returns x such that: (x*e) mod m == 1 + """ + x1, x2 = 1, 0 + a, b = e, m + while b > 0: + q, r = divmod(a, b) + xn = x1 - q * x2 + a, b, x1, x2 = b, r, x2, xn + return x1 % m + + +def rsa_crt_iqmp(p, q): + """ + Compute the CRT (q ** -1) % p value from RSA primes p and q. + """ + return _modinv(q, p) + + +def rsa_crt_dmp1(private_exponent, p): + """ + Compute the CRT private_exponent % (p - 1) value from the RSA + private_exponent (d) and p. + """ + return private_exponent % (p - 1) + + +def rsa_crt_dmq1(private_exponent, q): + """ + Compute the CRT private_exponent % (q - 1) value from the RSA + private_exponent (d) and q. + """ + return private_exponent % (q - 1) + + +# Controls the number of iterations rsa_recover_prime_factors will perform +# to obtain the prime factors. Each iteration increments by 2 so the actual +# maximum attempts is half this number. +_MAX_RECOVERY_ATTEMPTS = 1000 + + +def rsa_recover_prime_factors(n, e, d): + """ + Compute factors p and q from the private exponent d. We assume that n has + no more than two factors. This function is adapted from code in PyCrypto. + """ + # See 8.2.2(i) in Handbook of Applied Cryptography. + ktot = d * e - 1 + # The quantity d*e-1 is a multiple of phi(n), even, + # and can be represented as t*2^s. + t = ktot + while t % 2 == 0: + t = t // 2 + # Cycle through all multiplicative inverses in Zn. + # The algorithm is non-deterministic, but there is a 50% chance + # any candidate a leads to successful factoring. + # See "Digitalized Signatures and Public Key Functions as Intractable + # as Factorization", M. Rabin, 1979 + spotted = False + a = 2 + while not spotted and a < _MAX_RECOVERY_ATTEMPTS: + k = t + # Cycle through all values a^{t*2^i}=a^k + while k < ktot: + cand = pow(a, k, n) + # Check if a^k is a non-trivial root of unity (mod n) + if cand != 1 and cand != (n - 1) and pow(cand, 2, n) == 1: + # We have found a number such that (cand-1)(cand+1)=0 (mod n). + # Either of the terms divides n. + p = gcd(cand + 1, n) + spotted = True + break + k *= 2 + # This value was not any good... let's try another! + a += 2 + if not spotted: + raise ValueError("Unable to compute factors p and q from exponent d.") + # Found ! + q, r = divmod(n, p) + assert r == 0 + p, q = sorted((p, q), reverse=True) + return (p, q) + + +class RSAPrivateNumbers(object): + def __init__(self, p, q, d, dmp1, dmq1, iqmp, public_numbers): + if ( + not isinstance(p, six.integer_types) + or not isinstance(q, six.integer_types) + or not isinstance(d, six.integer_types) + or not isinstance(dmp1, six.integer_types) + or not isinstance(dmq1, six.integer_types) + or not isinstance(iqmp, six.integer_types) + ): + raise TypeError( + "RSAPrivateNumbers p, q, d, dmp1, dmq1, iqmp arguments must" + " all be an integers." + ) + + if not isinstance(public_numbers, RSAPublicNumbers): + raise TypeError( + "RSAPrivateNumbers public_numbers must be an RSAPublicNumbers" + " instance." + ) + + self._p = p + self._q = q + self._d = d + self._dmp1 = dmp1 + self._dmq1 = dmq1 + self._iqmp = iqmp + self._public_numbers = public_numbers + + p = utils.read_only_property("_p") + q = utils.read_only_property("_q") + d = utils.read_only_property("_d") + dmp1 = utils.read_only_property("_dmp1") + dmq1 = utils.read_only_property("_dmq1") + iqmp = utils.read_only_property("_iqmp") + public_numbers = utils.read_only_property("_public_numbers") + + def private_key(self, backend=None): + backend = _get_backend(backend) + return backend.load_rsa_private_numbers(self) + + def __eq__(self, other): + if not isinstance(other, RSAPrivateNumbers): + return NotImplemented + + return ( + self.p == other.p + and self.q == other.q + and self.d == other.d + and self.dmp1 == other.dmp1 + and self.dmq1 == other.dmq1 + and self.iqmp == other.iqmp + and self.public_numbers == other.public_numbers + ) + + def __ne__(self, other): + return not self == other + + def __hash__(self): + return hash( + ( + self.p, + self.q, + self.d, + self.dmp1, + self.dmq1, + self.iqmp, + self.public_numbers, + ) + ) + + +class RSAPublicNumbers(object): + def __init__(self, e, n): + if not isinstance(e, six.integer_types) or not isinstance( + n, six.integer_types + ): + raise TypeError("RSAPublicNumbers arguments must be integers.") + + self._e = e + self._n = n + + e = utils.read_only_property("_e") + n = utils.read_only_property("_n") + + def public_key(self, backend=None): + backend = _get_backend(backend) + return backend.load_rsa_public_numbers(self) + + def __repr__(self): + return "".format(self) + + def __eq__(self, other): + if not isinstance(other, RSAPublicNumbers): + return NotImplemented + + return self.e == other.e and self.n == other.n + + def __ne__(self, other): + return not self == other + + def __hash__(self): + return hash((self.e, self.n)) diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/asymmetric/utils.py b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/asymmetric/utils.py new file mode 100644 index 0000000..5f9b677 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/asymmetric/utils.py @@ -0,0 +1,41 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function + +from cryptography import utils +from cryptography.hazmat._der import ( + DERReader, + INTEGER, + SEQUENCE, + encode_der, + encode_der_integer, +) +from cryptography.hazmat.primitives import hashes + + +def decode_dss_signature(signature): + with DERReader(signature).read_single_element(SEQUENCE) as seq: + r = seq.read_element(INTEGER).as_integer() + s = seq.read_element(INTEGER).as_integer() + return r, s + + +def encode_dss_signature(r, s): + return encode_der( + SEQUENCE, + encode_der(INTEGER, encode_der_integer(r)), + encode_der(INTEGER, encode_der_integer(s)), + ) + + +class Prehashed(object): + def __init__(self, algorithm): + if not isinstance(algorithm, hashes.HashAlgorithm): + raise TypeError("Expected instance of HashAlgorithm.") + + self._algorithm = algorithm + self._digest_size = algorithm.digest_size + + digest_size = utils.read_only_property("_digest_size") diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/asymmetric/x25519.py b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/asymmetric/x25519.py new file mode 100644 index 0000000..fc63691 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/asymmetric/x25519.py @@ -0,0 +1,76 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function + +import abc + +import six + +from cryptography.exceptions import UnsupportedAlgorithm, _Reasons + + +@six.add_metaclass(abc.ABCMeta) +class X25519PublicKey(object): + @classmethod + def from_public_bytes(cls, data): + from cryptography.hazmat.backends.openssl.backend import backend + + if not backend.x25519_supported(): + raise UnsupportedAlgorithm( + "X25519 is not supported by this version of OpenSSL.", + _Reasons.UNSUPPORTED_EXCHANGE_ALGORITHM, + ) + + return backend.x25519_load_public_bytes(data) + + @abc.abstractmethod + def public_bytes(self, encoding, format): + """ + The serialized bytes of the public key. + """ + + +@six.add_metaclass(abc.ABCMeta) +class X25519PrivateKey(object): + @classmethod + def generate(cls): + from cryptography.hazmat.backends.openssl.backend import backend + + if not backend.x25519_supported(): + raise UnsupportedAlgorithm( + "X25519 is not supported by this version of OpenSSL.", + _Reasons.UNSUPPORTED_EXCHANGE_ALGORITHM, + ) + return backend.x25519_generate_key() + + @classmethod + def from_private_bytes(cls, data): + from cryptography.hazmat.backends.openssl.backend import backend + + if not backend.x25519_supported(): + raise UnsupportedAlgorithm( + "X25519 is not supported by this version of OpenSSL.", + _Reasons.UNSUPPORTED_EXCHANGE_ALGORITHM, + ) + + return backend.x25519_load_private_bytes(data) + + @abc.abstractmethod + def public_key(self): + """ + The serialized bytes of the public key. + """ + + @abc.abstractmethod + def private_bytes(self, encoding, format, encryption_algorithm): + """ + The serialized bytes of the private key. + """ + + @abc.abstractmethod + def exchange(self, peer_public_key): + """ + Performs a key exchange operation using the provided peer's public key. + """ diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/asymmetric/x448.py b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/asymmetric/x448.py new file mode 100644 index 0000000..3ac067b --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/asymmetric/x448.py @@ -0,0 +1,76 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function + +import abc + +import six + +from cryptography.exceptions import UnsupportedAlgorithm, _Reasons + + +@six.add_metaclass(abc.ABCMeta) +class X448PublicKey(object): + @classmethod + def from_public_bytes(cls, data): + from cryptography.hazmat.backends.openssl.backend import backend + + if not backend.x448_supported(): + raise UnsupportedAlgorithm( + "X448 is not supported by this version of OpenSSL.", + _Reasons.UNSUPPORTED_EXCHANGE_ALGORITHM, + ) + + return backend.x448_load_public_bytes(data) + + @abc.abstractmethod + def public_bytes(self, encoding, format): + """ + The serialized bytes of the public key. + """ + + +@six.add_metaclass(abc.ABCMeta) +class X448PrivateKey(object): + @classmethod + def generate(cls): + from cryptography.hazmat.backends.openssl.backend import backend + + if not backend.x448_supported(): + raise UnsupportedAlgorithm( + "X448 is not supported by this version of OpenSSL.", + _Reasons.UNSUPPORTED_EXCHANGE_ALGORITHM, + ) + return backend.x448_generate_key() + + @classmethod + def from_private_bytes(cls, data): + from cryptography.hazmat.backends.openssl.backend import backend + + if not backend.x448_supported(): + raise UnsupportedAlgorithm( + "X448 is not supported by this version of OpenSSL.", + _Reasons.UNSUPPORTED_EXCHANGE_ALGORITHM, + ) + + return backend.x448_load_private_bytes(data) + + @abc.abstractmethod + def public_key(self): + """ + The serialized bytes of the public key. + """ + + @abc.abstractmethod + def private_bytes(self, encoding, format, encryption_algorithm): + """ + The serialized bytes of the private key. + """ + + @abc.abstractmethod + def exchange(self, peer_public_key): + """ + Performs a key exchange operation using the provided peer's public key. + """ diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/ciphers/__init__.py b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/ciphers/__init__.py new file mode 100644 index 0000000..4380f72 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/ciphers/__init__.py @@ -0,0 +1,26 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function + +from cryptography.hazmat.primitives.ciphers.base import ( + AEADCipherContext, + AEADDecryptionContext, + AEADEncryptionContext, + BlockCipherAlgorithm, + Cipher, + CipherAlgorithm, + CipherContext, +) + + +__all__ = [ + "Cipher", + "CipherAlgorithm", + "BlockCipherAlgorithm", + "CipherContext", + "AEADCipherContext", + "AEADDecryptionContext", + "AEADEncryptionContext", +] diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/ciphers/__pycache__/__init__.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/ciphers/__pycache__/__init__.cpython-39.pyc new file mode 100644 index 0000000..83b633b Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/ciphers/__pycache__/__init__.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/ciphers/__pycache__/aead.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/ciphers/__pycache__/aead.cpython-39.pyc new file mode 100644 index 0000000..4c7a5f3 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/ciphers/__pycache__/aead.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/ciphers/__pycache__/algorithms.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/ciphers/__pycache__/algorithms.cpython-39.pyc new file mode 100644 index 0000000..8c3a934 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/ciphers/__pycache__/algorithms.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/ciphers/__pycache__/base.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/ciphers/__pycache__/base.cpython-39.pyc new file mode 100644 index 0000000..5c244f8 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/ciphers/__pycache__/base.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/ciphers/__pycache__/modes.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/ciphers/__pycache__/modes.cpython-39.pyc new file mode 100644 index 0000000..671c028 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/ciphers/__pycache__/modes.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/ciphers/aead.py b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/ciphers/aead.py new file mode 100644 index 0000000..c8c9395 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/ciphers/aead.py @@ -0,0 +1,174 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function + +import os + +from cryptography import exceptions, utils +from cryptography.hazmat.backends.openssl import aead +from cryptography.hazmat.backends.openssl.backend import backend + + +class ChaCha20Poly1305(object): + _MAX_SIZE = 2 ** 32 + + def __init__(self, key): + if not backend.aead_cipher_supported(self): + raise exceptions.UnsupportedAlgorithm( + "ChaCha20Poly1305 is not supported by this version of OpenSSL", + exceptions._Reasons.UNSUPPORTED_CIPHER, + ) + utils._check_byteslike("key", key) + + if len(key) != 32: + raise ValueError("ChaCha20Poly1305 key must be 32 bytes.") + + self._key = key + + @classmethod + def generate_key(cls): + return os.urandom(32) + + def encrypt(self, nonce, data, associated_data): + if associated_data is None: + associated_data = b"" + + if len(data) > self._MAX_SIZE or len(associated_data) > self._MAX_SIZE: + # This is OverflowError to match what cffi would raise + raise OverflowError( + "Data or associated data too long. Max 2**32 bytes" + ) + + self._check_params(nonce, data, associated_data) + return aead._encrypt(backend, self, nonce, data, associated_data, 16) + + def decrypt(self, nonce, data, associated_data): + if associated_data is None: + associated_data = b"" + + self._check_params(nonce, data, associated_data) + return aead._decrypt(backend, self, nonce, data, associated_data, 16) + + def _check_params(self, nonce, data, associated_data): + utils._check_byteslike("nonce", nonce) + utils._check_bytes("data", data) + utils._check_bytes("associated_data", associated_data) + if len(nonce) != 12: + raise ValueError("Nonce must be 12 bytes") + + +class AESCCM(object): + _MAX_SIZE = 2 ** 32 + + def __init__(self, key, tag_length=16): + utils._check_byteslike("key", key) + if len(key) not in (16, 24, 32): + raise ValueError("AESCCM key must be 128, 192, or 256 bits.") + + self._key = key + if not isinstance(tag_length, int): + raise TypeError("tag_length must be an integer") + + if tag_length not in (4, 6, 8, 10, 12, 14, 16): + raise ValueError("Invalid tag_length") + + self._tag_length = tag_length + + @classmethod + def generate_key(cls, bit_length): + if not isinstance(bit_length, int): + raise TypeError("bit_length must be an integer") + + if bit_length not in (128, 192, 256): + raise ValueError("bit_length must be 128, 192, or 256") + + return os.urandom(bit_length // 8) + + def encrypt(self, nonce, data, associated_data): + if associated_data is None: + associated_data = b"" + + if len(data) > self._MAX_SIZE or len(associated_data) > self._MAX_SIZE: + # This is OverflowError to match what cffi would raise + raise OverflowError( + "Data or associated data too long. Max 2**32 bytes" + ) + + self._check_params(nonce, data, associated_data) + self._validate_lengths(nonce, len(data)) + return aead._encrypt( + backend, self, nonce, data, associated_data, self._tag_length + ) + + def decrypt(self, nonce, data, associated_data): + if associated_data is None: + associated_data = b"" + + self._check_params(nonce, data, associated_data) + return aead._decrypt( + backend, self, nonce, data, associated_data, self._tag_length + ) + + def _validate_lengths(self, nonce, data_len): + # For information about computing this, see + # https://tools.ietf.org/html/rfc3610#section-2.1 + l_val = 15 - len(nonce) + if 2 ** (8 * l_val) < data_len: + raise ValueError("Data too long for nonce") + + def _check_params(self, nonce, data, associated_data): + utils._check_byteslike("nonce", nonce) + utils._check_bytes("data", data) + utils._check_bytes("associated_data", associated_data) + if not 7 <= len(nonce) <= 13: + raise ValueError("Nonce must be between 7 and 13 bytes") + + +class AESGCM(object): + _MAX_SIZE = 2 ** 32 + + def __init__(self, key): + utils._check_byteslike("key", key) + if len(key) not in (16, 24, 32): + raise ValueError("AESGCM key must be 128, 192, or 256 bits.") + + self._key = key + + @classmethod + def generate_key(cls, bit_length): + if not isinstance(bit_length, int): + raise TypeError("bit_length must be an integer") + + if bit_length not in (128, 192, 256): + raise ValueError("bit_length must be 128, 192, or 256") + + return os.urandom(bit_length // 8) + + def encrypt(self, nonce, data, associated_data): + if associated_data is None: + associated_data = b"" + + if len(data) > self._MAX_SIZE or len(associated_data) > self._MAX_SIZE: + # This is OverflowError to match what cffi would raise + raise OverflowError( + "Data or associated data too long. Max 2**32 bytes" + ) + + self._check_params(nonce, data, associated_data) + return aead._encrypt(backend, self, nonce, data, associated_data, 16) + + def decrypt(self, nonce, data, associated_data): + if associated_data is None: + associated_data = b"" + + self._check_params(nonce, data, associated_data) + return aead._decrypt(backend, self, nonce, data, associated_data, 16) + + def _check_params(self, nonce, data, associated_data): + utils._check_byteslike("nonce", nonce) + utils._check_bytes("data", data) + utils._check_bytes("associated_data", associated_data) + if len(nonce) < 8 or len(nonce) > 128: + raise ValueError("Nonce must be between 8 and 128 bytes") diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/ciphers/algorithms.py b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/ciphers/algorithms.py new file mode 100644 index 0000000..8072ced --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/ciphers/algorithms.py @@ -0,0 +1,170 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function + +from cryptography import utils +from cryptography.hazmat.primitives.ciphers import ( + BlockCipherAlgorithm, + CipherAlgorithm, +) +from cryptography.hazmat.primitives.ciphers.modes import ModeWithNonce + + +def _verify_key_size(algorithm, key): + # Verify that the key is instance of bytes + utils._check_byteslike("key", key) + + # Verify that the key size matches the expected key size + if len(key) * 8 not in algorithm.key_sizes: + raise ValueError( + "Invalid key size ({}) for {}.".format( + len(key) * 8, algorithm.name + ) + ) + return key + + +@utils.register_interface(BlockCipherAlgorithm) +@utils.register_interface(CipherAlgorithm) +class AES(object): + name = "AES" + block_size = 128 + # 512 added to support AES-256-XTS, which uses 512-bit keys + key_sizes = frozenset([128, 192, 256, 512]) + + def __init__(self, key): + self.key = _verify_key_size(self, key) + + @property + def key_size(self): + return len(self.key) * 8 + + +@utils.register_interface(BlockCipherAlgorithm) +@utils.register_interface(CipherAlgorithm) +class Camellia(object): + name = "camellia" + block_size = 128 + key_sizes = frozenset([128, 192, 256]) + + def __init__(self, key): + self.key = _verify_key_size(self, key) + + @property + def key_size(self): + return len(self.key) * 8 + + +@utils.register_interface(BlockCipherAlgorithm) +@utils.register_interface(CipherAlgorithm) +class TripleDES(object): + name = "3DES" + block_size = 64 + key_sizes = frozenset([64, 128, 192]) + + def __init__(self, key): + if len(key) == 8: + key += key + key + elif len(key) == 16: + key += key[:8] + self.key = _verify_key_size(self, key) + + @property + def key_size(self): + return len(self.key) * 8 + + +@utils.register_interface(BlockCipherAlgorithm) +@utils.register_interface(CipherAlgorithm) +class Blowfish(object): + name = "Blowfish" + block_size = 64 + key_sizes = frozenset(range(32, 449, 8)) + + def __init__(self, key): + self.key = _verify_key_size(self, key) + + @property + def key_size(self): + return len(self.key) * 8 + + +@utils.register_interface(BlockCipherAlgorithm) +@utils.register_interface(CipherAlgorithm) +class CAST5(object): + name = "CAST5" + block_size = 64 + key_sizes = frozenset(range(40, 129, 8)) + + def __init__(self, key): + self.key = _verify_key_size(self, key) + + @property + def key_size(self): + return len(self.key) * 8 + + +@utils.register_interface(CipherAlgorithm) +class ARC4(object): + name = "RC4" + key_sizes = frozenset([40, 56, 64, 80, 128, 160, 192, 256]) + + def __init__(self, key): + self.key = _verify_key_size(self, key) + + @property + def key_size(self): + return len(self.key) * 8 + + +@utils.register_interface(CipherAlgorithm) +class IDEA(object): + name = "IDEA" + block_size = 64 + key_sizes = frozenset([128]) + + def __init__(self, key): + self.key = _verify_key_size(self, key) + + @property + def key_size(self): + return len(self.key) * 8 + + +@utils.register_interface(BlockCipherAlgorithm) +@utils.register_interface(CipherAlgorithm) +class SEED(object): + name = "SEED" + block_size = 128 + key_sizes = frozenset([128]) + + def __init__(self, key): + self.key = _verify_key_size(self, key) + + @property + def key_size(self): + return len(self.key) * 8 + + +@utils.register_interface(CipherAlgorithm) +@utils.register_interface(ModeWithNonce) +class ChaCha20(object): + name = "ChaCha20" + key_sizes = frozenset([256]) + + def __init__(self, key, nonce): + self.key = _verify_key_size(self, key) + utils._check_byteslike("nonce", nonce) + + if len(nonce) != 16: + raise ValueError("nonce must be 128-bits (16 bytes)") + + self._nonce = nonce + + nonce = utils.read_only_property("_nonce") + + @property + def key_size(self): + return len(self.key) * 8 diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/ciphers/base.py b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/ciphers/base.py new file mode 100644 index 0000000..dae425a --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/ciphers/base.py @@ -0,0 +1,241 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function + +import abc + +import six + +from cryptography import utils +from cryptography.exceptions import ( + AlreadyFinalized, + AlreadyUpdated, + NotYetFinalized, + UnsupportedAlgorithm, + _Reasons, +) +from cryptography.hazmat.backends import _get_backend +from cryptography.hazmat.backends.interfaces import CipherBackend +from cryptography.hazmat.primitives.ciphers import modes + + +@six.add_metaclass(abc.ABCMeta) +class CipherAlgorithm(object): + @abc.abstractproperty + def name(self): + """ + A string naming this mode (e.g. "AES", "Camellia"). + """ + + @abc.abstractproperty + def key_size(self): + """ + The size of the key being used as an integer in bits (e.g. 128, 256). + """ + + +@six.add_metaclass(abc.ABCMeta) +class BlockCipherAlgorithm(object): + @abc.abstractproperty + def block_size(self): + """ + The size of a block as an integer in bits (e.g. 64, 128). + """ + + +@six.add_metaclass(abc.ABCMeta) +class CipherContext(object): + @abc.abstractmethod + def update(self, data): + """ + Processes the provided bytes through the cipher and returns the results + as bytes. + """ + + @abc.abstractmethod + def update_into(self, data, buf): + """ + Processes the provided bytes and writes the resulting data into the + provided buffer. Returns the number of bytes written. + """ + + @abc.abstractmethod + def finalize(self): + """ + Returns the results of processing the final block as bytes. + """ + + +@six.add_metaclass(abc.ABCMeta) +class AEADCipherContext(object): + @abc.abstractmethod + def authenticate_additional_data(self, data): + """ + Authenticates the provided bytes. + """ + + +@six.add_metaclass(abc.ABCMeta) +class AEADDecryptionContext(object): + @abc.abstractmethod + def finalize_with_tag(self, tag): + """ + Returns the results of processing the final block as bytes and allows + delayed passing of the authentication tag. + """ + + +@six.add_metaclass(abc.ABCMeta) +class AEADEncryptionContext(object): + @abc.abstractproperty + def tag(self): + """ + Returns tag bytes. This is only available after encryption is + finalized. + """ + + +class Cipher(object): + def __init__(self, algorithm, mode, backend=None): + backend = _get_backend(backend) + if not isinstance(backend, CipherBackend): + raise UnsupportedAlgorithm( + "Backend object does not implement CipherBackend.", + _Reasons.BACKEND_MISSING_INTERFACE, + ) + + if not isinstance(algorithm, CipherAlgorithm): + raise TypeError("Expected interface of CipherAlgorithm.") + + if mode is not None: + mode.validate_for_algorithm(algorithm) + + self.algorithm = algorithm + self.mode = mode + self._backend = backend + + def encryptor(self): + if isinstance(self.mode, modes.ModeWithAuthenticationTag): + if self.mode.tag is not None: + raise ValueError( + "Authentication tag must be None when encrypting." + ) + ctx = self._backend.create_symmetric_encryption_ctx( + self.algorithm, self.mode + ) + return self._wrap_ctx(ctx, encrypt=True) + + def decryptor(self): + ctx = self._backend.create_symmetric_decryption_ctx( + self.algorithm, self.mode + ) + return self._wrap_ctx(ctx, encrypt=False) + + def _wrap_ctx(self, ctx, encrypt): + if isinstance(self.mode, modes.ModeWithAuthenticationTag): + if encrypt: + return _AEADEncryptionContext(ctx) + else: + return _AEADCipherContext(ctx) + else: + return _CipherContext(ctx) + + +@utils.register_interface(CipherContext) +class _CipherContext(object): + def __init__(self, ctx): + self._ctx = ctx + + def update(self, data): + if self._ctx is None: + raise AlreadyFinalized("Context was already finalized.") + return self._ctx.update(data) + + def update_into(self, data, buf): + if self._ctx is None: + raise AlreadyFinalized("Context was already finalized.") + return self._ctx.update_into(data, buf) + + def finalize(self): + if self._ctx is None: + raise AlreadyFinalized("Context was already finalized.") + data = self._ctx.finalize() + self._ctx = None + return data + + +@utils.register_interface(AEADCipherContext) +@utils.register_interface(CipherContext) +@utils.register_interface(AEADDecryptionContext) +class _AEADCipherContext(object): + def __init__(self, ctx): + self._ctx = ctx + self._bytes_processed = 0 + self._aad_bytes_processed = 0 + self._tag = None + self._updated = False + + def _check_limit(self, data_size): + if self._ctx is None: + raise AlreadyFinalized("Context was already finalized.") + self._updated = True + self._bytes_processed += data_size + if self._bytes_processed > self._ctx._mode._MAX_ENCRYPTED_BYTES: + raise ValueError( + "{} has a maximum encrypted byte limit of {}".format( + self._ctx._mode.name, self._ctx._mode._MAX_ENCRYPTED_BYTES + ) + ) + + def update(self, data): + self._check_limit(len(data)) + return self._ctx.update(data) + + def update_into(self, data, buf): + self._check_limit(len(data)) + return self._ctx.update_into(data, buf) + + def finalize(self): + if self._ctx is None: + raise AlreadyFinalized("Context was already finalized.") + data = self._ctx.finalize() + self._tag = self._ctx.tag + self._ctx = None + return data + + def finalize_with_tag(self, tag): + if self._ctx is None: + raise AlreadyFinalized("Context was already finalized.") + data = self._ctx.finalize_with_tag(tag) + self._tag = self._ctx.tag + self._ctx = None + return data + + def authenticate_additional_data(self, data): + if self._ctx is None: + raise AlreadyFinalized("Context was already finalized.") + if self._updated: + raise AlreadyUpdated("Update has been called on this context.") + + self._aad_bytes_processed += len(data) + if self._aad_bytes_processed > self._ctx._mode._MAX_AAD_BYTES: + raise ValueError( + "{} has a maximum AAD byte limit of {}".format( + self._ctx._mode.name, self._ctx._mode._MAX_AAD_BYTES + ) + ) + + self._ctx.authenticate_additional_data(data) + + +@utils.register_interface(AEADEncryptionContext) +class _AEADEncryptionContext(_AEADCipherContext): + @property + def tag(self): + if self._ctx is not None: + raise NotYetFinalized( + "You must finalize encryption before " "getting the tag." + ) + return self._tag diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/ciphers/modes.py b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/ciphers/modes.py new file mode 100644 index 0000000..0ba0f2b --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/ciphers/modes.py @@ -0,0 +1,225 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function + +import abc + +import six + +from cryptography import utils + + +@six.add_metaclass(abc.ABCMeta) +class Mode(object): + @abc.abstractproperty + def name(self): + """ + A string naming this mode (e.g. "ECB", "CBC"). + """ + + @abc.abstractmethod + def validate_for_algorithm(self, algorithm): + """ + Checks that all the necessary invariants of this (mode, algorithm) + combination are met. + """ + + +@six.add_metaclass(abc.ABCMeta) +class ModeWithInitializationVector(object): + @abc.abstractproperty + def initialization_vector(self): + """ + The value of the initialization vector for this mode as bytes. + """ + + +@six.add_metaclass(abc.ABCMeta) +class ModeWithTweak(object): + @abc.abstractproperty + def tweak(self): + """ + The value of the tweak for this mode as bytes. + """ + + +@six.add_metaclass(abc.ABCMeta) +class ModeWithNonce(object): + @abc.abstractproperty + def nonce(self): + """ + The value of the nonce for this mode as bytes. + """ + + +@six.add_metaclass(abc.ABCMeta) +class ModeWithAuthenticationTag(object): + @abc.abstractproperty + def tag(self): + """ + The value of the tag supplied to the constructor of this mode. + """ + + +def _check_aes_key_length(self, algorithm): + if algorithm.key_size > 256 and algorithm.name == "AES": + raise ValueError( + "Only 128, 192, and 256 bit keys are allowed for this AES mode" + ) + + +def _check_iv_length(self, algorithm): + if len(self.initialization_vector) * 8 != algorithm.block_size: + raise ValueError( + "Invalid IV size ({}) for {}.".format( + len(self.initialization_vector), self.name + ) + ) + + +def _check_iv_and_key_length(self, algorithm): + _check_aes_key_length(self, algorithm) + _check_iv_length(self, algorithm) + + +@utils.register_interface(Mode) +@utils.register_interface(ModeWithInitializationVector) +class CBC(object): + name = "CBC" + + def __init__(self, initialization_vector): + utils._check_byteslike("initialization_vector", initialization_vector) + self._initialization_vector = initialization_vector + + initialization_vector = utils.read_only_property("_initialization_vector") + validate_for_algorithm = _check_iv_and_key_length + + +@utils.register_interface(Mode) +@utils.register_interface(ModeWithTweak) +class XTS(object): + name = "XTS" + + def __init__(self, tweak): + utils._check_byteslike("tweak", tweak) + + if len(tweak) != 16: + raise ValueError("tweak must be 128-bits (16 bytes)") + + self._tweak = tweak + + tweak = utils.read_only_property("_tweak") + + def validate_for_algorithm(self, algorithm): + if algorithm.key_size not in (256, 512): + raise ValueError( + "The XTS specification requires a 256-bit key for AES-128-XTS" + " and 512-bit key for AES-256-XTS" + ) + + +@utils.register_interface(Mode) +class ECB(object): + name = "ECB" + + validate_for_algorithm = _check_aes_key_length + + +@utils.register_interface(Mode) +@utils.register_interface(ModeWithInitializationVector) +class OFB(object): + name = "OFB" + + def __init__(self, initialization_vector): + utils._check_byteslike("initialization_vector", initialization_vector) + self._initialization_vector = initialization_vector + + initialization_vector = utils.read_only_property("_initialization_vector") + validate_for_algorithm = _check_iv_and_key_length + + +@utils.register_interface(Mode) +@utils.register_interface(ModeWithInitializationVector) +class CFB(object): + name = "CFB" + + def __init__(self, initialization_vector): + utils._check_byteslike("initialization_vector", initialization_vector) + self._initialization_vector = initialization_vector + + initialization_vector = utils.read_only_property("_initialization_vector") + validate_for_algorithm = _check_iv_and_key_length + + +@utils.register_interface(Mode) +@utils.register_interface(ModeWithInitializationVector) +class CFB8(object): + name = "CFB8" + + def __init__(self, initialization_vector): + utils._check_byteslike("initialization_vector", initialization_vector) + self._initialization_vector = initialization_vector + + initialization_vector = utils.read_only_property("_initialization_vector") + validate_for_algorithm = _check_iv_and_key_length + + +@utils.register_interface(Mode) +@utils.register_interface(ModeWithNonce) +class CTR(object): + name = "CTR" + + def __init__(self, nonce): + utils._check_byteslike("nonce", nonce) + self._nonce = nonce + + nonce = utils.read_only_property("_nonce") + + def validate_for_algorithm(self, algorithm): + _check_aes_key_length(self, algorithm) + if len(self.nonce) * 8 != algorithm.block_size: + raise ValueError( + "Invalid nonce size ({}) for {}.".format( + len(self.nonce), self.name + ) + ) + + +@utils.register_interface(Mode) +@utils.register_interface(ModeWithInitializationVector) +@utils.register_interface(ModeWithAuthenticationTag) +class GCM(object): + name = "GCM" + _MAX_ENCRYPTED_BYTES = (2 ** 39 - 256) // 8 + _MAX_AAD_BYTES = (2 ** 64) // 8 + + def __init__(self, initialization_vector, tag=None, min_tag_length=16): + # OpenSSL 3.0.0 constrains GCM IVs to [64, 1024] bits inclusive + # This is a sane limit anyway so we'll enforce it here. + utils._check_byteslike("initialization_vector", initialization_vector) + if len(initialization_vector) < 8 or len(initialization_vector) > 128: + raise ValueError( + "initialization_vector must be between 8 and 128 bytes (64 " + "and 1024 bits)." + ) + self._initialization_vector = initialization_vector + if tag is not None: + utils._check_bytes("tag", tag) + if min_tag_length < 4: + raise ValueError("min_tag_length must be >= 4") + if len(tag) < min_tag_length: + raise ValueError( + "Authentication tag must be {} bytes or longer.".format( + min_tag_length + ) + ) + self._tag = tag + self._min_tag_length = min_tag_length + + tag = utils.read_only_property("_tag") + initialization_vector = utils.read_only_property("_initialization_vector") + + def validate_for_algorithm(self, algorithm): + _check_aes_key_length(self, algorithm) diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/cmac.py b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/cmac.py new file mode 100644 index 0000000..bf962c9 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/cmac.py @@ -0,0 +1,64 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function + +from cryptography import utils +from cryptography.exceptions import ( + AlreadyFinalized, + UnsupportedAlgorithm, + _Reasons, +) +from cryptography.hazmat.backends import _get_backend +from cryptography.hazmat.backends.interfaces import CMACBackend +from cryptography.hazmat.primitives import ciphers + + +class CMAC(object): + def __init__(self, algorithm, backend=None, ctx=None): + backend = _get_backend(backend) + if not isinstance(backend, CMACBackend): + raise UnsupportedAlgorithm( + "Backend object does not implement CMACBackend.", + _Reasons.BACKEND_MISSING_INTERFACE, + ) + + if not isinstance(algorithm, ciphers.BlockCipherAlgorithm): + raise TypeError("Expected instance of BlockCipherAlgorithm.") + self._algorithm = algorithm + + self._backend = backend + if ctx is None: + self._ctx = self._backend.create_cmac_ctx(self._algorithm) + else: + self._ctx = ctx + + def update(self, data): + if self._ctx is None: + raise AlreadyFinalized("Context was already finalized.") + + utils._check_bytes("data", data) + self._ctx.update(data) + + def finalize(self): + if self._ctx is None: + raise AlreadyFinalized("Context was already finalized.") + digest = self._ctx.finalize() + self._ctx = None + return digest + + def verify(self, signature): + utils._check_bytes("signature", signature) + if self._ctx is None: + raise AlreadyFinalized("Context was already finalized.") + + ctx, self._ctx = self._ctx, None + ctx.verify(signature) + + def copy(self): + if self._ctx is None: + raise AlreadyFinalized("Context was already finalized.") + return CMAC( + self._algorithm, backend=self._backend, ctx=self._ctx.copy() + ) diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/constant_time.py b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/constant_time.py new file mode 100644 index 0000000..7f41b9e --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/constant_time.py @@ -0,0 +1,14 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function + +import hmac + + +def bytes_eq(a, b): + if not isinstance(a, bytes) or not isinstance(b, bytes): + raise TypeError("a and b must be bytes.") + + return hmac.compare_digest(a, b) diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/hashes.py b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/hashes.py new file mode 100644 index 0000000..18e2bab --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/hashes.py @@ -0,0 +1,259 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function + +import abc + +import six + +from cryptography import utils +from cryptography.exceptions import ( + AlreadyFinalized, + UnsupportedAlgorithm, + _Reasons, +) +from cryptography.hazmat.backends import _get_backend +from cryptography.hazmat.backends.interfaces import HashBackend + + +@six.add_metaclass(abc.ABCMeta) +class HashAlgorithm(object): + @abc.abstractproperty + def name(self): + """ + A string naming this algorithm (e.g. "sha256", "md5"). + """ + + @abc.abstractproperty + def digest_size(self): + """ + The size of the resulting digest in bytes. + """ + + +@six.add_metaclass(abc.ABCMeta) +class HashContext(object): + @abc.abstractproperty + def algorithm(self): + """ + A HashAlgorithm that will be used by this context. + """ + + @abc.abstractmethod + def update(self, data): + """ + Processes the provided bytes through the hash. + """ + + @abc.abstractmethod + def finalize(self): + """ + Finalizes the hash context and returns the hash digest as bytes. + """ + + @abc.abstractmethod + def copy(self): + """ + Return a HashContext that is a copy of the current context. + """ + + +@six.add_metaclass(abc.ABCMeta) +class ExtendableOutputFunction(object): + """ + An interface for extendable output functions. + """ + + +@utils.register_interface(HashContext) +class Hash(object): + def __init__(self, algorithm, backend=None, ctx=None): + backend = _get_backend(backend) + if not isinstance(backend, HashBackend): + raise UnsupportedAlgorithm( + "Backend object does not implement HashBackend.", + _Reasons.BACKEND_MISSING_INTERFACE, + ) + + if not isinstance(algorithm, HashAlgorithm): + raise TypeError("Expected instance of hashes.HashAlgorithm.") + self._algorithm = algorithm + + self._backend = backend + + if ctx is None: + self._ctx = self._backend.create_hash_ctx(self.algorithm) + else: + self._ctx = ctx + + algorithm = utils.read_only_property("_algorithm") + + def update(self, data): + if self._ctx is None: + raise AlreadyFinalized("Context was already finalized.") + utils._check_byteslike("data", data) + self._ctx.update(data) + + def copy(self): + if self._ctx is None: + raise AlreadyFinalized("Context was already finalized.") + return Hash( + self.algorithm, backend=self._backend, ctx=self._ctx.copy() + ) + + def finalize(self): + if self._ctx is None: + raise AlreadyFinalized("Context was already finalized.") + digest = self._ctx.finalize() + self._ctx = None + return digest + + +@utils.register_interface(HashAlgorithm) +class SHA1(object): + name = "sha1" + digest_size = 20 + block_size = 64 + + +@utils.register_interface(HashAlgorithm) +class SHA512_224(object): # noqa: N801 + name = "sha512-224" + digest_size = 28 + block_size = 128 + + +@utils.register_interface(HashAlgorithm) +class SHA512_256(object): # noqa: N801 + name = "sha512-256" + digest_size = 32 + block_size = 128 + + +@utils.register_interface(HashAlgorithm) +class SHA224(object): + name = "sha224" + digest_size = 28 + block_size = 64 + + +@utils.register_interface(HashAlgorithm) +class SHA256(object): + name = "sha256" + digest_size = 32 + block_size = 64 + + +@utils.register_interface(HashAlgorithm) +class SHA384(object): + name = "sha384" + digest_size = 48 + block_size = 128 + + +@utils.register_interface(HashAlgorithm) +class SHA512(object): + name = "sha512" + digest_size = 64 + block_size = 128 + + +@utils.register_interface(HashAlgorithm) +class SHA3_224(object): # noqa: N801 + name = "sha3-224" + digest_size = 28 + + +@utils.register_interface(HashAlgorithm) +class SHA3_256(object): # noqa: N801 + name = "sha3-256" + digest_size = 32 + + +@utils.register_interface(HashAlgorithm) +class SHA3_384(object): # noqa: N801 + name = "sha3-384" + digest_size = 48 + + +@utils.register_interface(HashAlgorithm) +class SHA3_512(object): # noqa: N801 + name = "sha3-512" + digest_size = 64 + + +@utils.register_interface(HashAlgorithm) +@utils.register_interface(ExtendableOutputFunction) +class SHAKE128(object): + name = "shake128" + + def __init__(self, digest_size): + if not isinstance(digest_size, six.integer_types): + raise TypeError("digest_size must be an integer") + + if digest_size < 1: + raise ValueError("digest_size must be a positive integer") + + self._digest_size = digest_size + + digest_size = utils.read_only_property("_digest_size") + + +@utils.register_interface(HashAlgorithm) +@utils.register_interface(ExtendableOutputFunction) +class SHAKE256(object): + name = "shake256" + + def __init__(self, digest_size): + if not isinstance(digest_size, six.integer_types): + raise TypeError("digest_size must be an integer") + + if digest_size < 1: + raise ValueError("digest_size must be a positive integer") + + self._digest_size = digest_size + + digest_size = utils.read_only_property("_digest_size") + + +@utils.register_interface(HashAlgorithm) +class MD5(object): + name = "md5" + digest_size = 16 + block_size = 64 + + +@utils.register_interface(HashAlgorithm) +class BLAKE2b(object): + name = "blake2b" + _max_digest_size = 64 + _min_digest_size = 1 + block_size = 128 + + def __init__(self, digest_size): + + if digest_size != 64: + raise ValueError("Digest size must be 64") + + self._digest_size = digest_size + + digest_size = utils.read_only_property("_digest_size") + + +@utils.register_interface(HashAlgorithm) +class BLAKE2s(object): + name = "blake2s" + block_size = 64 + _max_digest_size = 32 + _min_digest_size = 1 + + def __init__(self, digest_size): + + if digest_size != 32: + raise ValueError("Digest size must be 32") + + self._digest_size = digest_size + + digest_size = utils.read_only_property("_digest_size") diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/hmac.py b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/hmac.py new file mode 100644 index 0000000..8c421dc --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/hmac.py @@ -0,0 +1,70 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function + +from cryptography import utils +from cryptography.exceptions import ( + AlreadyFinalized, + UnsupportedAlgorithm, + _Reasons, +) +from cryptography.hazmat.backends import _get_backend +from cryptography.hazmat.backends.interfaces import HMACBackend +from cryptography.hazmat.primitives import hashes + + +@utils.register_interface(hashes.HashContext) +class HMAC(object): + def __init__(self, key, algorithm, backend=None, ctx=None): + backend = _get_backend(backend) + if not isinstance(backend, HMACBackend): + raise UnsupportedAlgorithm( + "Backend object does not implement HMACBackend.", + _Reasons.BACKEND_MISSING_INTERFACE, + ) + + if not isinstance(algorithm, hashes.HashAlgorithm): + raise TypeError("Expected instance of hashes.HashAlgorithm.") + self._algorithm = algorithm + + self._backend = backend + self._key = key + if ctx is None: + self._ctx = self._backend.create_hmac_ctx(key, self.algorithm) + else: + self._ctx = ctx + + algorithm = utils.read_only_property("_algorithm") + + def update(self, data): + if self._ctx is None: + raise AlreadyFinalized("Context was already finalized.") + utils._check_byteslike("data", data) + self._ctx.update(data) + + def copy(self): + if self._ctx is None: + raise AlreadyFinalized("Context was already finalized.") + return HMAC( + self._key, + self.algorithm, + backend=self._backend, + ctx=self._ctx.copy(), + ) + + def finalize(self): + if self._ctx is None: + raise AlreadyFinalized("Context was already finalized.") + digest = self._ctx.finalize() + self._ctx = None + return digest + + def verify(self, signature): + utils._check_bytes("signature", signature) + if self._ctx is None: + raise AlreadyFinalized("Context was already finalized.") + + ctx, self._ctx = self._ctx, None + ctx.verify(signature) diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/kdf/__init__.py b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/kdf/__init__.py new file mode 100644 index 0000000..2d0724e --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/kdf/__init__.py @@ -0,0 +1,26 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function + +import abc + +import six + + +@six.add_metaclass(abc.ABCMeta) +class KeyDerivationFunction(object): + @abc.abstractmethod + def derive(self, key_material): + """ + Deterministically generates and returns a new key based on the existing + key material. + """ + + @abc.abstractmethod + def verify(self, key_material, expected_key): + """ + Checks whether the key generated by the key material matches the + expected derived key. Raises an exception if they do not match. + """ diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/kdf/__pycache__/__init__.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/kdf/__pycache__/__init__.cpython-39.pyc new file mode 100644 index 0000000..216a38b Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/kdf/__pycache__/__init__.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/kdf/__pycache__/concatkdf.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/kdf/__pycache__/concatkdf.cpython-39.pyc new file mode 100644 index 0000000..b130f04 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/kdf/__pycache__/concatkdf.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/kdf/__pycache__/hkdf.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/kdf/__pycache__/hkdf.cpython-39.pyc new file mode 100644 index 0000000..a12b89d Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/kdf/__pycache__/hkdf.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/kdf/__pycache__/kbkdf.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/kdf/__pycache__/kbkdf.cpython-39.pyc new file mode 100644 index 0000000..ea63d4a Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/kdf/__pycache__/kbkdf.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/kdf/__pycache__/pbkdf2.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/kdf/__pycache__/pbkdf2.cpython-39.pyc new file mode 100644 index 0000000..9c3a1cf Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/kdf/__pycache__/pbkdf2.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/kdf/__pycache__/scrypt.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/kdf/__pycache__/scrypt.cpython-39.pyc new file mode 100644 index 0000000..1941263 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/kdf/__pycache__/scrypt.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/kdf/__pycache__/x963kdf.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/kdf/__pycache__/x963kdf.cpython-39.pyc new file mode 100644 index 0000000..c707bd2 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/kdf/__pycache__/x963kdf.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/kdf/concatkdf.py b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/kdf/concatkdf.py new file mode 100644 index 0000000..7cc0324 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/kdf/concatkdf.py @@ -0,0 +1,131 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function + +import struct + +from cryptography import utils +from cryptography.exceptions import ( + AlreadyFinalized, + InvalidKey, + UnsupportedAlgorithm, + _Reasons, +) +from cryptography.hazmat.backends import _get_backend +from cryptography.hazmat.backends.interfaces import HMACBackend +from cryptography.hazmat.backends.interfaces import HashBackend +from cryptography.hazmat.primitives import constant_time, hashes, hmac +from cryptography.hazmat.primitives.kdf import KeyDerivationFunction + + +def _int_to_u32be(n): + return struct.pack(">I", n) + + +def _common_args_checks(algorithm, length, otherinfo): + max_length = algorithm.digest_size * (2 ** 32 - 1) + if length > max_length: + raise ValueError( + "Can not derive keys larger than {} bits.".format(max_length) + ) + if otherinfo is not None: + utils._check_bytes("otherinfo", otherinfo) + + +def _concatkdf_derive(key_material, length, auxfn, otherinfo): + utils._check_byteslike("key_material", key_material) + output = [b""] + outlen = 0 + counter = 1 + + while length > outlen: + h = auxfn() + h.update(_int_to_u32be(counter)) + h.update(key_material) + h.update(otherinfo) + output.append(h.finalize()) + outlen += len(output[-1]) + counter += 1 + + return b"".join(output)[:length] + + +@utils.register_interface(KeyDerivationFunction) +class ConcatKDFHash(object): + def __init__(self, algorithm, length, otherinfo, backend=None): + backend = _get_backend(backend) + + _common_args_checks(algorithm, length, otherinfo) + self._algorithm = algorithm + self._length = length + self._otherinfo = otherinfo + if self._otherinfo is None: + self._otherinfo = b"" + + if not isinstance(backend, HashBackend): + raise UnsupportedAlgorithm( + "Backend object does not implement HashBackend.", + _Reasons.BACKEND_MISSING_INTERFACE, + ) + self._backend = backend + self._used = False + + def _hash(self): + return hashes.Hash(self._algorithm, self._backend) + + def derive(self, key_material): + if self._used: + raise AlreadyFinalized + self._used = True + return _concatkdf_derive( + key_material, self._length, self._hash, self._otherinfo + ) + + def verify(self, key_material, expected_key): + if not constant_time.bytes_eq(self.derive(key_material), expected_key): + raise InvalidKey + + +@utils.register_interface(KeyDerivationFunction) +class ConcatKDFHMAC(object): + def __init__(self, algorithm, length, salt, otherinfo, backend=None): + backend = _get_backend(backend) + + _common_args_checks(algorithm, length, otherinfo) + self._algorithm = algorithm + self._length = length + self._otherinfo = otherinfo + if self._otherinfo is None: + self._otherinfo = b"" + + if salt is None: + salt = b"\x00" * algorithm.block_size + else: + utils._check_bytes("salt", salt) + + self._salt = salt + + if not isinstance(backend, HMACBackend): + raise UnsupportedAlgorithm( + "Backend object does not implement HMACBackend.", + _Reasons.BACKEND_MISSING_INTERFACE, + ) + self._backend = backend + self._used = False + + def _hmac(self): + return hmac.HMAC(self._salt, self._algorithm, self._backend) + + def derive(self, key_material): + if self._used: + raise AlreadyFinalized + self._used = True + return _concatkdf_derive( + key_material, self._length, self._hmac, self._otherinfo + ) + + def verify(self, key_material, expected_key): + if not constant_time.bytes_eq(self.derive(key_material), expected_key): + raise InvalidKey diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/kdf/hkdf.py b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/kdf/hkdf.py new file mode 100644 index 0000000..9bb6bc2 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/kdf/hkdf.py @@ -0,0 +1,115 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function + +import six + +from cryptography import utils +from cryptography.exceptions import ( + AlreadyFinalized, + InvalidKey, + UnsupportedAlgorithm, + _Reasons, +) +from cryptography.hazmat.backends import _get_backend +from cryptography.hazmat.backends.interfaces import HMACBackend +from cryptography.hazmat.primitives import constant_time, hmac +from cryptography.hazmat.primitives.kdf import KeyDerivationFunction + + +@utils.register_interface(KeyDerivationFunction) +class HKDF(object): + def __init__(self, algorithm, length, salt, info, backend=None): + backend = _get_backend(backend) + if not isinstance(backend, HMACBackend): + raise UnsupportedAlgorithm( + "Backend object does not implement HMACBackend.", + _Reasons.BACKEND_MISSING_INTERFACE, + ) + + self._algorithm = algorithm + + if salt is None: + salt = b"\x00" * self._algorithm.digest_size + else: + utils._check_bytes("salt", salt) + + self._salt = salt + + self._backend = backend + + self._hkdf_expand = HKDFExpand(self._algorithm, length, info, backend) + + def _extract(self, key_material): + h = hmac.HMAC(self._salt, self._algorithm, backend=self._backend) + h.update(key_material) + return h.finalize() + + def derive(self, key_material): + utils._check_byteslike("key_material", key_material) + return self._hkdf_expand.derive(self._extract(key_material)) + + def verify(self, key_material, expected_key): + if not constant_time.bytes_eq(self.derive(key_material), expected_key): + raise InvalidKey + + +@utils.register_interface(KeyDerivationFunction) +class HKDFExpand(object): + def __init__(self, algorithm, length, info, backend=None): + backend = _get_backend(backend) + if not isinstance(backend, HMACBackend): + raise UnsupportedAlgorithm( + "Backend object does not implement HMACBackend.", + _Reasons.BACKEND_MISSING_INTERFACE, + ) + + self._algorithm = algorithm + + self._backend = backend + + max_length = 255 * algorithm.digest_size + + if length > max_length: + raise ValueError( + "Can not derive keys larger than {} octets.".format(max_length) + ) + + self._length = length + + if info is None: + info = b"" + else: + utils._check_bytes("info", info) + + self._info = info + + self._used = False + + def _expand(self, key_material): + output = [b""] + counter = 1 + + while self._algorithm.digest_size * (len(output) - 1) < self._length: + h = hmac.HMAC(key_material, self._algorithm, backend=self._backend) + h.update(output[-1]) + h.update(self._info) + h.update(six.int2byte(counter)) + output.append(h.finalize()) + counter += 1 + + return b"".join(output)[: self._length] + + def derive(self, key_material): + utils._check_byteslike("key_material", key_material) + if self._used: + raise AlreadyFinalized + + self._used = True + return self._expand(key_material) + + def verify(self, key_material, expected_key): + if not constant_time.bytes_eq(self.derive(key_material), expected_key): + raise InvalidKey diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/kdf/kbkdf.py b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/kdf/kbkdf.py new file mode 100644 index 0000000..8643370 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/kdf/kbkdf.py @@ -0,0 +1,162 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function + +from enum import Enum + +from six.moves import range + +from cryptography import utils +from cryptography.exceptions import ( + AlreadyFinalized, + InvalidKey, + UnsupportedAlgorithm, + _Reasons, +) +from cryptography.hazmat.backends import _get_backend +from cryptography.hazmat.backends.interfaces import HMACBackend +from cryptography.hazmat.primitives import constant_time, hashes, hmac +from cryptography.hazmat.primitives.kdf import KeyDerivationFunction + + +class Mode(Enum): + CounterMode = "ctr" + + +class CounterLocation(Enum): + BeforeFixed = "before_fixed" + AfterFixed = "after_fixed" + + +@utils.register_interface(KeyDerivationFunction) +class KBKDFHMAC(object): + def __init__( + self, + algorithm, + mode, + length, + rlen, + llen, + location, + label, + context, + fixed, + backend=None, + ): + backend = _get_backend(backend) + if not isinstance(backend, HMACBackend): + raise UnsupportedAlgorithm( + "Backend object does not implement HMACBackend.", + _Reasons.BACKEND_MISSING_INTERFACE, + ) + + if not isinstance(algorithm, hashes.HashAlgorithm): + raise UnsupportedAlgorithm( + "Algorithm supplied is not a supported hash algorithm.", + _Reasons.UNSUPPORTED_HASH, + ) + + if not backend.hmac_supported(algorithm): + raise UnsupportedAlgorithm( + "Algorithm supplied is not a supported hmac algorithm.", + _Reasons.UNSUPPORTED_HASH, + ) + + if not isinstance(mode, Mode): + raise TypeError("mode must be of type Mode") + + if not isinstance(location, CounterLocation): + raise TypeError("location must be of type CounterLocation") + + if (label or context) and fixed: + raise ValueError( + "When supplying fixed data, " "label and context are ignored." + ) + + if rlen is None or not self._valid_byte_length(rlen): + raise ValueError("rlen must be between 1 and 4") + + if llen is None and fixed is None: + raise ValueError("Please specify an llen") + + if llen is not None and not isinstance(llen, int): + raise TypeError("llen must be an integer") + + if label is None: + label = b"" + + if context is None: + context = b"" + + utils._check_bytes("label", label) + utils._check_bytes("context", context) + self._algorithm = algorithm + self._mode = mode + self._length = length + self._rlen = rlen + self._llen = llen + self._location = location + self._label = label + self._context = context + self._backend = backend + self._used = False + self._fixed_data = fixed + + def _valid_byte_length(self, value): + if not isinstance(value, int): + raise TypeError("value must be of type int") + + value_bin = utils.int_to_bytes(1, value) + if not 1 <= len(value_bin) <= 4: + return False + return True + + def derive(self, key_material): + if self._used: + raise AlreadyFinalized + + utils._check_byteslike("key_material", key_material) + self._used = True + + # inverse floor division (equivalent to ceiling) + rounds = -(-self._length // self._algorithm.digest_size) + + output = [b""] + + # For counter mode, the number of iterations shall not be + # larger than 2^r-1, where r <= 32 is the binary length of the counter + # This ensures that the counter values used as an input to the + # PRF will not repeat during a particular call to the KDF function. + r_bin = utils.int_to_bytes(1, self._rlen) + if rounds > pow(2, len(r_bin) * 8) - 1: + raise ValueError("There are too many iterations.") + + for i in range(1, rounds + 1): + h = hmac.HMAC(key_material, self._algorithm, backend=self._backend) + + counter = utils.int_to_bytes(i, self._rlen) + if self._location == CounterLocation.BeforeFixed: + h.update(counter) + + h.update(self._generate_fixed_input()) + + if self._location == CounterLocation.AfterFixed: + h.update(counter) + + output.append(h.finalize()) + + return b"".join(output)[: self._length] + + def _generate_fixed_input(self): + if self._fixed_data and isinstance(self._fixed_data, bytes): + return self._fixed_data + + l_val = utils.int_to_bytes(self._length * 8, self._llen) + + return b"".join([self._label, b"\x00", self._context, l_val]) + + def verify(self, key_material, expected_key): + if not constant_time.bytes_eq(self.derive(key_material), expected_key): + raise InvalidKey diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/kdf/pbkdf2.py b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/kdf/pbkdf2.py new file mode 100644 index 0000000..5b67d48 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/kdf/pbkdf2.py @@ -0,0 +1,62 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function + +from cryptography import utils +from cryptography.exceptions import ( + AlreadyFinalized, + InvalidKey, + UnsupportedAlgorithm, + _Reasons, +) +from cryptography.hazmat.backends import _get_backend +from cryptography.hazmat.backends.interfaces import PBKDF2HMACBackend +from cryptography.hazmat.primitives import constant_time +from cryptography.hazmat.primitives.kdf import KeyDerivationFunction + + +@utils.register_interface(KeyDerivationFunction) +class PBKDF2HMAC(object): + def __init__(self, algorithm, length, salt, iterations, backend=None): + backend = _get_backend(backend) + if not isinstance(backend, PBKDF2HMACBackend): + raise UnsupportedAlgorithm( + "Backend object does not implement PBKDF2HMACBackend.", + _Reasons.BACKEND_MISSING_INTERFACE, + ) + + if not backend.pbkdf2_hmac_supported(algorithm): + raise UnsupportedAlgorithm( + "{} is not supported for PBKDF2 by this backend.".format( + algorithm.name + ), + _Reasons.UNSUPPORTED_HASH, + ) + self._used = False + self._algorithm = algorithm + self._length = length + utils._check_bytes("salt", salt) + self._salt = salt + self._iterations = iterations + self._backend = backend + + def derive(self, key_material): + if self._used: + raise AlreadyFinalized("PBKDF2 instances can only be used once.") + self._used = True + + utils._check_byteslike("key_material", key_material) + return self._backend.derive_pbkdf2_hmac( + self._algorithm, + self._length, + self._salt, + self._iterations, + key_material, + ) + + def verify(self, key_material, expected_key): + derived_key = self.derive(key_material) + if not constant_time.bytes_eq(derived_key, expected_key): + raise InvalidKey("Keys do not match.") diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/kdf/scrypt.py b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/kdf/scrypt.py new file mode 100644 index 0000000..f028646 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/kdf/scrypt.py @@ -0,0 +1,68 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function + +import sys + +from cryptography import utils +from cryptography.exceptions import ( + AlreadyFinalized, + InvalidKey, + UnsupportedAlgorithm, + _Reasons, +) +from cryptography.hazmat.backends import _get_backend +from cryptography.hazmat.backends.interfaces import ScryptBackend +from cryptography.hazmat.primitives import constant_time +from cryptography.hazmat.primitives.kdf import KeyDerivationFunction + + +# This is used by the scrypt tests to skip tests that require more memory +# than the MEM_LIMIT +_MEM_LIMIT = sys.maxsize // 2 + + +@utils.register_interface(KeyDerivationFunction) +class Scrypt(object): + def __init__(self, salt, length, n, r, p, backend=None): + backend = _get_backend(backend) + if not isinstance(backend, ScryptBackend): + raise UnsupportedAlgorithm( + "Backend object does not implement ScryptBackend.", + _Reasons.BACKEND_MISSING_INTERFACE, + ) + + self._length = length + utils._check_bytes("salt", salt) + if n < 2 or (n & (n - 1)) != 0: + raise ValueError("n must be greater than 1 and be a power of 2.") + + if r < 1: + raise ValueError("r must be greater than or equal to 1.") + + if p < 1: + raise ValueError("p must be greater than or equal to 1.") + + self._used = False + self._salt = salt + self._n = n + self._r = r + self._p = p + self._backend = backend + + def derive(self, key_material): + if self._used: + raise AlreadyFinalized("Scrypt instances can only be used once.") + self._used = True + + utils._check_byteslike("key_material", key_material) + return self._backend.derive_scrypt( + key_material, self._salt, self._length, self._n, self._r, self._p + ) + + def verify(self, key_material, expected_key): + derived_key = self.derive(key_material) + if not constant_time.bytes_eq(derived_key, expected_key): + raise InvalidKey("Keys do not match.") diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/kdf/x963kdf.py b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/kdf/x963kdf.py new file mode 100644 index 0000000..1898d52 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/kdf/x963kdf.py @@ -0,0 +1,74 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function + +import struct + +from cryptography import utils +from cryptography.exceptions import ( + AlreadyFinalized, + InvalidKey, + UnsupportedAlgorithm, + _Reasons, +) +from cryptography.hazmat.backends import _get_backend +from cryptography.hazmat.backends.interfaces import HashBackend +from cryptography.hazmat.primitives import constant_time, hashes +from cryptography.hazmat.primitives.kdf import KeyDerivationFunction + + +def _int_to_u32be(n): + return struct.pack(">I", n) + + +@utils.register_interface(KeyDerivationFunction) +class X963KDF(object): + def __init__(self, algorithm, length, sharedinfo, backend=None): + backend = _get_backend(backend) + + max_len = algorithm.digest_size * (2 ** 32 - 1) + if length > max_len: + raise ValueError( + "Can not derive keys larger than {} bits.".format(max_len) + ) + if sharedinfo is not None: + utils._check_bytes("sharedinfo", sharedinfo) + + self._algorithm = algorithm + self._length = length + self._sharedinfo = sharedinfo + + if not isinstance(backend, HashBackend): + raise UnsupportedAlgorithm( + "Backend object does not implement HashBackend.", + _Reasons.BACKEND_MISSING_INTERFACE, + ) + self._backend = backend + self._used = False + + def derive(self, key_material): + if self._used: + raise AlreadyFinalized + self._used = True + utils._check_byteslike("key_material", key_material) + output = [b""] + outlen = 0 + counter = 1 + + while self._length > outlen: + h = hashes.Hash(self._algorithm, self._backend) + h.update(key_material) + h.update(_int_to_u32be(counter)) + if self._sharedinfo is not None: + h.update(self._sharedinfo) + output.append(h.finalize()) + outlen += len(output[-1]) + counter += 1 + + return b"".join(output)[: self._length] + + def verify(self, key_material, expected_key): + if not constant_time.bytes_eq(self.derive(key_material), expected_key): + raise InvalidKey diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/keywrap.py b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/keywrap.py new file mode 100644 index 0000000..2439caf --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/keywrap.py @@ -0,0 +1,161 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function + +import struct + +from cryptography.hazmat.backends import _get_backend +from cryptography.hazmat.primitives.ciphers import Cipher +from cryptography.hazmat.primitives.ciphers.algorithms import AES +from cryptography.hazmat.primitives.ciphers.modes import ECB +from cryptography.hazmat.primitives.constant_time import bytes_eq + + +def _wrap_core(wrapping_key, a, r, backend): + # RFC 3394 Key Wrap - 2.2.1 (index method) + encryptor = Cipher(AES(wrapping_key), ECB(), backend).encryptor() + n = len(r) + for j in range(6): + for i in range(n): + # every encryption operation is a discrete 16 byte chunk (because + # AES has a 128-bit block size) and since we're using ECB it is + # safe to reuse the encryptor for the entire operation + b = encryptor.update(a + r[i]) + # pack/unpack are safe as these are always 64-bit chunks + a = struct.pack( + ">Q", struct.unpack(">Q", b[:8])[0] ^ ((n * j) + i + 1) + ) + r[i] = b[-8:] + + assert encryptor.finalize() == b"" + + return a + b"".join(r) + + +def aes_key_wrap(wrapping_key, key_to_wrap, backend=None): + backend = _get_backend(backend) + if len(wrapping_key) not in [16, 24, 32]: + raise ValueError("The wrapping key must be a valid AES key length") + + if len(key_to_wrap) < 16: + raise ValueError("The key to wrap must be at least 16 bytes") + + if len(key_to_wrap) % 8 != 0: + raise ValueError("The key to wrap must be a multiple of 8 bytes") + + a = b"\xa6\xa6\xa6\xa6\xa6\xa6\xa6\xa6" + r = [key_to_wrap[i : i + 8] for i in range(0, len(key_to_wrap), 8)] + return _wrap_core(wrapping_key, a, r, backend) + + +def _unwrap_core(wrapping_key, a, r, backend): + # Implement RFC 3394 Key Unwrap - 2.2.2 (index method) + decryptor = Cipher(AES(wrapping_key), ECB(), backend).decryptor() + n = len(r) + for j in reversed(range(6)): + for i in reversed(range(n)): + # pack/unpack are safe as these are always 64-bit chunks + atr = ( + struct.pack( + ">Q", struct.unpack(">Q", a)[0] ^ ((n * j) + i + 1) + ) + + r[i] + ) + # every decryption operation is a discrete 16 byte chunk so + # it is safe to reuse the decryptor for the entire operation + b = decryptor.update(atr) + a = b[:8] + r[i] = b[-8:] + + assert decryptor.finalize() == b"" + return a, r + + +def aes_key_wrap_with_padding(wrapping_key, key_to_wrap, backend=None): + backend = _get_backend(backend) + if len(wrapping_key) not in [16, 24, 32]: + raise ValueError("The wrapping key must be a valid AES key length") + + aiv = b"\xA6\x59\x59\xA6" + struct.pack(">i", len(key_to_wrap)) + # pad the key to wrap if necessary + pad = (8 - (len(key_to_wrap) % 8)) % 8 + key_to_wrap = key_to_wrap + b"\x00" * pad + if len(key_to_wrap) == 8: + # RFC 5649 - 4.1 - exactly 8 octets after padding + encryptor = Cipher(AES(wrapping_key), ECB(), backend).encryptor() + b = encryptor.update(aiv + key_to_wrap) + assert encryptor.finalize() == b"" + return b + else: + r = [key_to_wrap[i : i + 8] for i in range(0, len(key_to_wrap), 8)] + return _wrap_core(wrapping_key, aiv, r, backend) + + +def aes_key_unwrap_with_padding(wrapping_key, wrapped_key, backend=None): + backend = _get_backend(backend) + if len(wrapped_key) < 16: + raise InvalidUnwrap("Must be at least 16 bytes") + + if len(wrapping_key) not in [16, 24, 32]: + raise ValueError("The wrapping key must be a valid AES key length") + + if len(wrapped_key) == 16: + # RFC 5649 - 4.2 - exactly two 64-bit blocks + decryptor = Cipher(AES(wrapping_key), ECB(), backend).decryptor() + b = decryptor.update(wrapped_key) + assert decryptor.finalize() == b"" + a = b[:8] + data = b[8:] + n = 1 + else: + r = [wrapped_key[i : i + 8] for i in range(0, len(wrapped_key), 8)] + encrypted_aiv = r.pop(0) + n = len(r) + a, r = _unwrap_core(wrapping_key, encrypted_aiv, r, backend) + data = b"".join(r) + + # 1) Check that MSB(32,A) = A65959A6. + # 2) Check that 8*(n-1) < LSB(32,A) <= 8*n. If so, let + # MLI = LSB(32,A). + # 3) Let b = (8*n)-MLI, and then check that the rightmost b octets of + # the output data are zero. + (mli,) = struct.unpack(">I", a[4:]) + b = (8 * n) - mli + if ( + not bytes_eq(a[:4], b"\xa6\x59\x59\xa6") + or not 8 * (n - 1) < mli <= 8 * n + or (b != 0 and not bytes_eq(data[-b:], b"\x00" * b)) + ): + raise InvalidUnwrap() + + if b == 0: + return data + else: + return data[:-b] + + +def aes_key_unwrap(wrapping_key, wrapped_key, backend=None): + backend = _get_backend(backend) + if len(wrapped_key) < 24: + raise InvalidUnwrap("Must be at least 24 bytes") + + if len(wrapped_key) % 8 != 0: + raise InvalidUnwrap("The wrapped key must be a multiple of 8 bytes") + + if len(wrapping_key) not in [16, 24, 32]: + raise ValueError("The wrapping key must be a valid AES key length") + + aiv = b"\xa6\xa6\xa6\xa6\xa6\xa6\xa6\xa6" + r = [wrapped_key[i : i + 8] for i in range(0, len(wrapped_key), 8)] + a = r.pop(0) + a, r = _unwrap_core(wrapping_key, a, r, backend) + if not bytes_eq(a, aiv): + raise InvalidUnwrap() + + return b"".join(r) + + +class InvalidUnwrap(Exception): + pass diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/padding.py b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/padding.py new file mode 100644 index 0000000..98abffb --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/padding.py @@ -0,0 +1,214 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function + +import abc + +import six + +from cryptography import utils +from cryptography.exceptions import AlreadyFinalized +from cryptography.hazmat.bindings._padding import lib + + +@six.add_metaclass(abc.ABCMeta) +class PaddingContext(object): + @abc.abstractmethod + def update(self, data): + """ + Pads the provided bytes and returns any available data as bytes. + """ + + @abc.abstractmethod + def finalize(self): + """ + Finalize the padding, returns bytes. + """ + + +def _byte_padding_check(block_size): + if not (0 <= block_size <= 2040): + raise ValueError("block_size must be in range(0, 2041).") + + if block_size % 8 != 0: + raise ValueError("block_size must be a multiple of 8.") + + +def _byte_padding_update(buffer_, data, block_size): + if buffer_ is None: + raise AlreadyFinalized("Context was already finalized.") + + utils._check_byteslike("data", data) + + # six.PY2: Only coerce non-bytes objects to avoid triggering bad behavior + # of future's newbytes type. Unconditionally call bytes() after Python 2 + # support is gone. + buffer_ += data if isinstance(data, bytes) else bytes(data) + + finished_blocks = len(buffer_) // (block_size // 8) + + result = buffer_[: finished_blocks * (block_size // 8)] + buffer_ = buffer_[finished_blocks * (block_size // 8) :] + + return buffer_, result + + +def _byte_padding_pad(buffer_, block_size, paddingfn): + if buffer_ is None: + raise AlreadyFinalized("Context was already finalized.") + + pad_size = block_size // 8 - len(buffer_) + return buffer_ + paddingfn(pad_size) + + +def _byte_unpadding_update(buffer_, data, block_size): + if buffer_ is None: + raise AlreadyFinalized("Context was already finalized.") + + utils._check_byteslike("data", data) + + # six.PY2: Only coerce non-bytes objects to avoid triggering bad behavior + # of future's newbytes type. Unconditionally call bytes() after Python 2 + # support is gone. + buffer_ += data if isinstance(data, bytes) else bytes(data) + + finished_blocks = max(len(buffer_) // (block_size // 8) - 1, 0) + + result = buffer_[: finished_blocks * (block_size // 8)] + buffer_ = buffer_[finished_blocks * (block_size // 8) :] + + return buffer_, result + + +def _byte_unpadding_check(buffer_, block_size, checkfn): + if buffer_ is None: + raise AlreadyFinalized("Context was already finalized.") + + if len(buffer_) != block_size // 8: + raise ValueError("Invalid padding bytes.") + + valid = checkfn(buffer_, block_size // 8) + + if not valid: + raise ValueError("Invalid padding bytes.") + + pad_size = six.indexbytes(buffer_, -1) + return buffer_[:-pad_size] + + +class PKCS7(object): + def __init__(self, block_size): + _byte_padding_check(block_size) + self.block_size = block_size + + def padder(self): + return _PKCS7PaddingContext(self.block_size) + + def unpadder(self): + return _PKCS7UnpaddingContext(self.block_size) + + +@utils.register_interface(PaddingContext) +class _PKCS7PaddingContext(object): + def __init__(self, block_size): + self.block_size = block_size + # TODO: more copies than necessary, we should use zero-buffer (#193) + self._buffer = b"" + + def update(self, data): + self._buffer, result = _byte_padding_update( + self._buffer, data, self.block_size + ) + return result + + def _padding(self, size): + return six.int2byte(size) * size + + def finalize(self): + result = _byte_padding_pad( + self._buffer, self.block_size, self._padding + ) + self._buffer = None + return result + + +@utils.register_interface(PaddingContext) +class _PKCS7UnpaddingContext(object): + def __init__(self, block_size): + self.block_size = block_size + # TODO: more copies than necessary, we should use zero-buffer (#193) + self._buffer = b"" + + def update(self, data): + self._buffer, result = _byte_unpadding_update( + self._buffer, data, self.block_size + ) + return result + + def finalize(self): + result = _byte_unpadding_check( + self._buffer, self.block_size, lib.Cryptography_check_pkcs7_padding + ) + self._buffer = None + return result + + +class ANSIX923(object): + def __init__(self, block_size): + _byte_padding_check(block_size) + self.block_size = block_size + + def padder(self): + return _ANSIX923PaddingContext(self.block_size) + + def unpadder(self): + return _ANSIX923UnpaddingContext(self.block_size) + + +@utils.register_interface(PaddingContext) +class _ANSIX923PaddingContext(object): + def __init__(self, block_size): + self.block_size = block_size + # TODO: more copies than necessary, we should use zero-buffer (#193) + self._buffer = b"" + + def update(self, data): + self._buffer, result = _byte_padding_update( + self._buffer, data, self.block_size + ) + return result + + def _padding(self, size): + return six.int2byte(0) * (size - 1) + six.int2byte(size) + + def finalize(self): + result = _byte_padding_pad( + self._buffer, self.block_size, self._padding + ) + self._buffer = None + return result + + +@utils.register_interface(PaddingContext) +class _ANSIX923UnpaddingContext(object): + def __init__(self, block_size): + self.block_size = block_size + # TODO: more copies than necessary, we should use zero-buffer (#193) + self._buffer = b"" + + def update(self, data): + self._buffer, result = _byte_unpadding_update( + self._buffer, data, self.block_size + ) + return result + + def finalize(self): + result = _byte_unpadding_check( + self._buffer, + self.block_size, + lib.Cryptography_check_ansix923_padding, + ) + self._buffer = None + return result diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/poly1305.py b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/poly1305.py new file mode 100644 index 0000000..6439686 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/poly1305.py @@ -0,0 +1,58 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function + + +from cryptography import utils +from cryptography.exceptions import ( + AlreadyFinalized, + UnsupportedAlgorithm, + _Reasons, +) + + +class Poly1305(object): + def __init__(self, key): + from cryptography.hazmat.backends.openssl.backend import backend + + if not backend.poly1305_supported(): + raise UnsupportedAlgorithm( + "poly1305 is not supported by this version of OpenSSL.", + _Reasons.UNSUPPORTED_MAC, + ) + self._ctx = backend.create_poly1305_ctx(key) + + def update(self, data): + if self._ctx is None: + raise AlreadyFinalized("Context was already finalized.") + utils._check_byteslike("data", data) + self._ctx.update(data) + + def finalize(self): + if self._ctx is None: + raise AlreadyFinalized("Context was already finalized.") + mac = self._ctx.finalize() + self._ctx = None + return mac + + def verify(self, tag): + utils._check_bytes("tag", tag) + if self._ctx is None: + raise AlreadyFinalized("Context was already finalized.") + + ctx, self._ctx = self._ctx, None + ctx.verify(tag) + + @classmethod + def generate_tag(cls, key, data): + p = Poly1305(key) + p.update(data) + return p.finalize() + + @classmethod + def verify_tag(cls, key, data, tag): + p = Poly1305(key) + p.update(data) + p.verify(tag) diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/serialization/__init__.py b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/serialization/__init__.py new file mode 100644 index 0000000..c2f9b01 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/serialization/__init__.py @@ -0,0 +1,44 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function + +from cryptography.hazmat.primitives.serialization.base import ( + BestAvailableEncryption, + Encoding, + KeySerializationEncryption, + NoEncryption, + ParameterFormat, + PrivateFormat, + PublicFormat, + load_der_parameters, + load_der_private_key, + load_der_public_key, + load_pem_parameters, + load_pem_private_key, + load_pem_public_key, +) +from cryptography.hazmat.primitives.serialization.ssh import ( + load_ssh_private_key, + load_ssh_public_key, +) + + +__all__ = [ + "load_der_parameters", + "load_der_private_key", + "load_der_public_key", + "load_pem_parameters", + "load_pem_private_key", + "load_pem_public_key", + "load_ssh_private_key", + "load_ssh_public_key", + "Encoding", + "PrivateFormat", + "PublicFormat", + "ParameterFormat", + "KeySerializationEncryption", + "BestAvailableEncryption", + "NoEncryption", +] diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/serialization/__pycache__/__init__.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/serialization/__pycache__/__init__.cpython-39.pyc new file mode 100644 index 0000000..c0e4d95 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/serialization/__pycache__/__init__.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/serialization/__pycache__/base.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/serialization/__pycache__/base.cpython-39.pyc new file mode 100644 index 0000000..0c6ef7d Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/serialization/__pycache__/base.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/serialization/__pycache__/pkcs12.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/serialization/__pycache__/pkcs12.cpython-39.pyc new file mode 100644 index 0000000..68f582e Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/serialization/__pycache__/pkcs12.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/serialization/__pycache__/pkcs7.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/serialization/__pycache__/pkcs7.cpython-39.pyc new file mode 100644 index 0000000..92aee21 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/serialization/__pycache__/pkcs7.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/serialization/__pycache__/ssh.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/serialization/__pycache__/ssh.cpython-39.pyc new file mode 100644 index 0000000..0c187c7 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/serialization/__pycache__/ssh.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/serialization/base.py b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/serialization/base.py new file mode 100644 index 0000000..fc27235 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/serialization/base.py @@ -0,0 +1,91 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function + +import abc +from enum import Enum + +import six + +from cryptography import utils +from cryptography.hazmat.backends import _get_backend + + +def load_pem_private_key(data, password, backend=None): + backend = _get_backend(backend) + return backend.load_pem_private_key(data, password) + + +def load_pem_public_key(data, backend=None): + backend = _get_backend(backend) + return backend.load_pem_public_key(data) + + +def load_pem_parameters(data, backend=None): + backend = _get_backend(backend) + return backend.load_pem_parameters(data) + + +def load_der_private_key(data, password, backend=None): + backend = _get_backend(backend) + return backend.load_der_private_key(data, password) + + +def load_der_public_key(data, backend=None): + backend = _get_backend(backend) + return backend.load_der_public_key(data) + + +def load_der_parameters(data, backend=None): + backend = _get_backend(backend) + return backend.load_der_parameters(data) + + +class Encoding(Enum): + PEM = "PEM" + DER = "DER" + OpenSSH = "OpenSSH" + Raw = "Raw" + X962 = "ANSI X9.62" + SMIME = "S/MIME" + + +class PrivateFormat(Enum): + PKCS8 = "PKCS8" + TraditionalOpenSSL = "TraditionalOpenSSL" + Raw = "Raw" + OpenSSH = "OpenSSH" + + +class PublicFormat(Enum): + SubjectPublicKeyInfo = "X.509 subjectPublicKeyInfo with PKCS#1" + PKCS1 = "Raw PKCS#1" + OpenSSH = "OpenSSH" + Raw = "Raw" + CompressedPoint = "X9.62 Compressed Point" + UncompressedPoint = "X9.62 Uncompressed Point" + + +class ParameterFormat(Enum): + PKCS3 = "PKCS3" + + +@six.add_metaclass(abc.ABCMeta) +class KeySerializationEncryption(object): + pass + + +@utils.register_interface(KeySerializationEncryption) +class BestAvailableEncryption(object): + def __init__(self, password): + if not isinstance(password, bytes) or len(password) == 0: + raise ValueError("Password must be 1 or more bytes.") + + self.password = password + + +@utils.register_interface(KeySerializationEncryption) +class NoEncryption(object): + pass diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/serialization/pkcs12.py b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/serialization/pkcs12.py new file mode 100644 index 0000000..201f329 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/serialization/pkcs12.py @@ -0,0 +1,50 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function + +from cryptography import x509 +from cryptography.hazmat.backends import _get_backend +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.asymmetric import dsa, ec, rsa + + +def load_key_and_certificates(data, password, backend=None): + backend = _get_backend(backend) + return backend.load_key_and_certificates_from_pkcs12(data, password) + + +def serialize_key_and_certificates(name, key, cert, cas, encryption_algorithm): + if key is not None and not isinstance( + key, + ( + rsa.RSAPrivateKeyWithSerialization, + dsa.DSAPrivateKeyWithSerialization, + ec.EllipticCurvePrivateKeyWithSerialization, + ), + ): + raise TypeError("Key must be RSA, DSA, or EllipticCurve private key.") + if cert is not None and not isinstance(cert, x509.Certificate): + raise TypeError("cert must be a certificate") + + if cas is not None: + cas = list(cas) + if not all(isinstance(val, x509.Certificate) for val in cas): + raise TypeError("all values in cas must be certificates") + + if not isinstance( + encryption_algorithm, serialization.KeySerializationEncryption + ): + raise TypeError( + "Key encryption algorithm must be a " + "KeySerializationEncryption instance" + ) + + if key is None and cert is None and not cas: + raise ValueError("You must supply at least one of key, cert, or cas") + + backend = _get_backend(None) + return backend.serialize_key_and_certificates_to_pkcs12( + name, key, cert, cas, encryption_algorithm + ) diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/serialization/pkcs7.py b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/serialization/pkcs7.py new file mode 100644 index 0000000..1e11e28 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/serialization/pkcs7.py @@ -0,0 +1,132 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function + +from enum import Enum + +from cryptography import x509 +from cryptography.hazmat.backends import _get_backend +from cryptography.hazmat.primitives import hashes, serialization +from cryptography.hazmat.primitives.asymmetric import ec, rsa +from cryptography.utils import _check_byteslike + + +def load_pem_pkcs7_certificates(data): + backend = _get_backend(None) + return backend.load_pem_pkcs7_certificates(data) + + +def load_der_pkcs7_certificates(data): + backend = _get_backend(None) + return backend.load_der_pkcs7_certificates(data) + + +class PKCS7SignatureBuilder(object): + def __init__(self, data=None, signers=[], additional_certs=[]): + self._data = data + self._signers = signers + self._additional_certs = additional_certs + + def set_data(self, data): + _check_byteslike("data", data) + if self._data is not None: + raise ValueError("data may only be set once") + + return PKCS7SignatureBuilder(data, self._signers) + + def add_signer(self, certificate, private_key, hash_algorithm): + if not isinstance( + hash_algorithm, + ( + hashes.SHA1, + hashes.SHA224, + hashes.SHA256, + hashes.SHA384, + hashes.SHA512, + ), + ): + raise TypeError( + "hash_algorithm must be one of hashes.SHA1, SHA224, " + "SHA256, SHA384, or SHA512" + ) + if not isinstance(certificate, x509.Certificate): + raise TypeError("certificate must be a x509.Certificate") + + if not isinstance( + private_key, (rsa.RSAPrivateKey, ec.EllipticCurvePrivateKey) + ): + raise TypeError("Only RSA & EC keys are supported at this time.") + + return PKCS7SignatureBuilder( + self._data, + self._signers + [(certificate, private_key, hash_algorithm)], + ) + + def add_certificate(self, certificate): + if not isinstance(certificate, x509.Certificate): + raise TypeError("certificate must be a x509.Certificate") + + return PKCS7SignatureBuilder( + self._data, self._signers, self._additional_certs + [certificate] + ) + + def sign(self, encoding, options, backend=None): + if len(self._signers) == 0: + raise ValueError("Must have at least one signer") + if self._data is None: + raise ValueError("You must add data to sign") + options = list(options) + if not all(isinstance(x, PKCS7Options) for x in options): + raise ValueError("options must be from the PKCS7Options enum") + if encoding not in ( + serialization.Encoding.PEM, + serialization.Encoding.DER, + serialization.Encoding.SMIME, + ): + raise ValueError( + "Must be PEM, DER, or SMIME from the Encoding enum" + ) + + # Text is a meaningless option unless it is accompanied by + # DetachedSignature + if ( + PKCS7Options.Text in options + and PKCS7Options.DetachedSignature not in options + ): + raise ValueError( + "When passing the Text option you must also pass " + "DetachedSignature" + ) + + if PKCS7Options.Text in options and encoding in ( + serialization.Encoding.DER, + serialization.Encoding.PEM, + ): + raise ValueError( + "The Text option is only available for SMIME serialization" + ) + + # No attributes implies no capabilities so we'll error if you try to + # pass both. + if ( + PKCS7Options.NoAttributes in options + and PKCS7Options.NoCapabilities in options + ): + raise ValueError( + "NoAttributes is a superset of NoCapabilities. Do not pass " + "both values." + ) + + backend = _get_backend(backend) + return backend.pkcs7_sign(self, encoding, options) + + +class PKCS7Options(Enum): + Text = "Add text/plain MIME type" + Binary = "Don't translate input data into canonical MIME format" + DetachedSignature = "Don't embed data in the PKCS7 structure" + NoCapabilities = "Don't embed SMIME capabilities" + NoAttributes = "Don't embed authenticatedAttributes" + NoCerts = "Don't embed signer certificate" diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/serialization/ssh.py b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/serialization/ssh.py new file mode 100644 index 0000000..5ecae59 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/serialization/ssh.py @@ -0,0 +1,683 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function + +import binascii +import os +import re +import struct + +import six + +from cryptography import utils +from cryptography.exceptions import UnsupportedAlgorithm +from cryptography.hazmat.backends import _get_backend +from cryptography.hazmat.primitives.asymmetric import dsa, ec, ed25519, rsa +from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes +from cryptography.hazmat.primitives.serialization import ( + Encoding, + NoEncryption, + PrivateFormat, + PublicFormat, +) + +try: + from bcrypt import kdf as _bcrypt_kdf + + _bcrypt_supported = True +except ImportError: + _bcrypt_supported = False + + def _bcrypt_kdf(*args, **kwargs): + raise UnsupportedAlgorithm("Need bcrypt module") + + +try: + from base64 import encodebytes as _base64_encode +except ImportError: + from base64 import encodestring as _base64_encode + +_SSH_ED25519 = b"ssh-ed25519" +_SSH_RSA = b"ssh-rsa" +_SSH_DSA = b"ssh-dss" +_ECDSA_NISTP256 = b"ecdsa-sha2-nistp256" +_ECDSA_NISTP384 = b"ecdsa-sha2-nistp384" +_ECDSA_NISTP521 = b"ecdsa-sha2-nistp521" +_CERT_SUFFIX = b"-cert-v01@openssh.com" + +_SSH_PUBKEY_RC = re.compile(br"\A(\S+)[ \t]+(\S+)") +_SK_MAGIC = b"openssh-key-v1\0" +_SK_START = b"-----BEGIN OPENSSH PRIVATE KEY-----" +_SK_END = b"-----END OPENSSH PRIVATE KEY-----" +_BCRYPT = b"bcrypt" +_NONE = b"none" +_DEFAULT_CIPHER = b"aes256-ctr" +_DEFAULT_ROUNDS = 16 +_MAX_PASSWORD = 72 + +# re is only way to work on bytes-like data +_PEM_RC = re.compile(_SK_START + b"(.*?)" + _SK_END, re.DOTALL) + +# padding for max blocksize +_PADDING = memoryview(bytearray(range(1, 1 + 16))) + +# ciphers that are actually used in key wrapping +_SSH_CIPHERS = { + b"aes256-ctr": (algorithms.AES, 32, modes.CTR, 16), + b"aes256-cbc": (algorithms.AES, 32, modes.CBC, 16), +} + +# map local curve name to key type +_ECDSA_KEY_TYPE = { + "secp256r1": _ECDSA_NISTP256, + "secp384r1": _ECDSA_NISTP384, + "secp521r1": _ECDSA_NISTP521, +} + +_U32 = struct.Struct(b">I") +_U64 = struct.Struct(b">Q") + + +def _ecdsa_key_type(public_key): + """Return SSH key_type and curve_name for private key.""" + curve = public_key.curve + if curve.name not in _ECDSA_KEY_TYPE: + raise ValueError( + "Unsupported curve for ssh private key: %r" % curve.name + ) + return _ECDSA_KEY_TYPE[curve.name] + + +def _ssh_pem_encode(data, prefix=_SK_START + b"\n", suffix=_SK_END + b"\n"): + return b"".join([prefix, _base64_encode(data), suffix]) + + +def _check_block_size(data, block_len): + """Require data to be full blocks""" + if not data or len(data) % block_len != 0: + raise ValueError("Corrupt data: missing padding") + + +def _check_empty(data): + """All data should have been parsed.""" + if data: + raise ValueError("Corrupt data: unparsed data") + + +def _init_cipher(ciphername, password, salt, rounds, backend): + """Generate key + iv and return cipher.""" + if not password: + raise ValueError("Key is password-protected.") + + algo, key_len, mode, iv_len = _SSH_CIPHERS[ciphername] + seed = _bcrypt_kdf(password, salt, key_len + iv_len, rounds, True) + return Cipher(algo(seed[:key_len]), mode(seed[key_len:]), backend) + + +def _get_u32(data): + """Uint32""" + if len(data) < 4: + raise ValueError("Invalid data") + return _U32.unpack(data[:4])[0], data[4:] + + +def _get_u64(data): + """Uint64""" + if len(data) < 8: + raise ValueError("Invalid data") + return _U64.unpack(data[:8])[0], data[8:] + + +def _get_sshstr(data): + """Bytes with u32 length prefix""" + n, data = _get_u32(data) + if n > len(data): + raise ValueError("Invalid data") + return data[:n], data[n:] + + +def _get_mpint(data): + """Big integer.""" + val, data = _get_sshstr(data) + if val and six.indexbytes(val, 0) > 0x7F: + raise ValueError("Invalid data") + return utils.int_from_bytes(val, "big"), data + + +def _to_mpint(val): + """Storage format for signed bigint.""" + if val < 0: + raise ValueError("negative mpint not allowed") + if not val: + return b"" + nbytes = (val.bit_length() + 8) // 8 + return utils.int_to_bytes(val, nbytes) + + +class _FragList(object): + """Build recursive structure without data copy.""" + + def __init__(self, init=None): + self.flist = [] + if init: + self.flist.extend(init) + + def put_raw(self, val): + """Add plain bytes""" + self.flist.append(val) + + def put_u32(self, val): + """Big-endian uint32""" + self.flist.append(_U32.pack(val)) + + def put_sshstr(self, val): + """Bytes prefixed with u32 length""" + if isinstance(val, (bytes, memoryview, bytearray)): + self.put_u32(len(val)) + self.flist.append(val) + else: + self.put_u32(val.size()) + self.flist.extend(val.flist) + + def put_mpint(self, val): + """Big-endian bigint prefixed with u32 length""" + self.put_sshstr(_to_mpint(val)) + + def size(self): + """Current number of bytes""" + return sum(map(len, self.flist)) + + def render(self, dstbuf, pos=0): + """Write into bytearray""" + for frag in self.flist: + flen = len(frag) + start, pos = pos, pos + flen + dstbuf[start:pos] = frag + return pos + + def tobytes(self): + """Return as bytes""" + buf = memoryview(bytearray(self.size())) + self.render(buf) + return buf.tobytes() + + +class _SSHFormatRSA(object): + """Format for RSA keys. + + Public: + mpint e, n + Private: + mpint n, e, d, iqmp, p, q + """ + + def get_public(self, data): + """RSA public fields""" + e, data = _get_mpint(data) + n, data = _get_mpint(data) + return (e, n), data + + def load_public(self, key_type, data, backend): + """Make RSA public key from data.""" + (e, n), data = self.get_public(data) + public_numbers = rsa.RSAPublicNumbers(e, n) + public_key = public_numbers.public_key(backend) + return public_key, data + + def load_private(self, data, pubfields, backend): + """Make RSA private key from data.""" + n, data = _get_mpint(data) + e, data = _get_mpint(data) + d, data = _get_mpint(data) + iqmp, data = _get_mpint(data) + p, data = _get_mpint(data) + q, data = _get_mpint(data) + + if (e, n) != pubfields: + raise ValueError("Corrupt data: rsa field mismatch") + dmp1 = rsa.rsa_crt_dmp1(d, p) + dmq1 = rsa.rsa_crt_dmq1(d, q) + public_numbers = rsa.RSAPublicNumbers(e, n) + private_numbers = rsa.RSAPrivateNumbers( + p, q, d, dmp1, dmq1, iqmp, public_numbers + ) + private_key = private_numbers.private_key(backend) + return private_key, data + + def encode_public(self, public_key, f_pub): + """Write RSA public key""" + pubn = public_key.public_numbers() + f_pub.put_mpint(pubn.e) + f_pub.put_mpint(pubn.n) + + def encode_private(self, private_key, f_priv): + """Write RSA private key""" + private_numbers = private_key.private_numbers() + public_numbers = private_numbers.public_numbers + + f_priv.put_mpint(public_numbers.n) + f_priv.put_mpint(public_numbers.e) + + f_priv.put_mpint(private_numbers.d) + f_priv.put_mpint(private_numbers.iqmp) + f_priv.put_mpint(private_numbers.p) + f_priv.put_mpint(private_numbers.q) + + +class _SSHFormatDSA(object): + """Format for DSA keys. + + Public: + mpint p, q, g, y + Private: + mpint p, q, g, y, x + """ + + def get_public(self, data): + """DSA public fields""" + p, data = _get_mpint(data) + q, data = _get_mpint(data) + g, data = _get_mpint(data) + y, data = _get_mpint(data) + return (p, q, g, y), data + + def load_public(self, key_type, data, backend): + """Make DSA public key from data.""" + (p, q, g, y), data = self.get_public(data) + parameter_numbers = dsa.DSAParameterNumbers(p, q, g) + public_numbers = dsa.DSAPublicNumbers(y, parameter_numbers) + self._validate(public_numbers) + public_key = public_numbers.public_key(backend) + return public_key, data + + def load_private(self, data, pubfields, backend): + """Make DSA private key from data.""" + (p, q, g, y), data = self.get_public(data) + x, data = _get_mpint(data) + + if (p, q, g, y) != pubfields: + raise ValueError("Corrupt data: dsa field mismatch") + parameter_numbers = dsa.DSAParameterNumbers(p, q, g) + public_numbers = dsa.DSAPublicNumbers(y, parameter_numbers) + self._validate(public_numbers) + private_numbers = dsa.DSAPrivateNumbers(x, public_numbers) + private_key = private_numbers.private_key(backend) + return private_key, data + + def encode_public(self, public_key, f_pub): + """Write DSA public key""" + public_numbers = public_key.public_numbers() + parameter_numbers = public_numbers.parameter_numbers + self._validate(public_numbers) + + f_pub.put_mpint(parameter_numbers.p) + f_pub.put_mpint(parameter_numbers.q) + f_pub.put_mpint(parameter_numbers.g) + f_pub.put_mpint(public_numbers.y) + + def encode_private(self, private_key, f_priv): + """Write DSA private key""" + self.encode_public(private_key.public_key(), f_priv) + f_priv.put_mpint(private_key.private_numbers().x) + + def _validate(self, public_numbers): + parameter_numbers = public_numbers.parameter_numbers + if parameter_numbers.p.bit_length() != 1024: + raise ValueError("SSH supports only 1024 bit DSA keys") + + +class _SSHFormatECDSA(object): + """Format for ECDSA keys. + + Public: + str curve + bytes point + Private: + str curve + bytes point + mpint secret + """ + + def __init__(self, ssh_curve_name, curve): + self.ssh_curve_name = ssh_curve_name + self.curve = curve + + def get_public(self, data): + """ECDSA public fields""" + curve, data = _get_sshstr(data) + point, data = _get_sshstr(data) + if curve != self.ssh_curve_name: + raise ValueError("Curve name mismatch") + if six.indexbytes(point, 0) != 4: + raise NotImplementedError("Need uncompressed point") + return (curve, point), data + + def load_public(self, key_type, data, backend): + """Make ECDSA public key from data.""" + (curve_name, point), data = self.get_public(data) + public_key = ec.EllipticCurvePublicKey.from_encoded_point( + self.curve, point.tobytes() + ) + return public_key, data + + def load_private(self, data, pubfields, backend): + """Make ECDSA private key from data.""" + (curve_name, point), data = self.get_public(data) + secret, data = _get_mpint(data) + + if (curve_name, point) != pubfields: + raise ValueError("Corrupt data: ecdsa field mismatch") + private_key = ec.derive_private_key(secret, self.curve, backend) + return private_key, data + + def encode_public(self, public_key, f_pub): + """Write ECDSA public key""" + point = public_key.public_bytes( + Encoding.X962, PublicFormat.UncompressedPoint + ) + f_pub.put_sshstr(self.ssh_curve_name) + f_pub.put_sshstr(point) + + def encode_private(self, private_key, f_priv): + """Write ECDSA private key""" + public_key = private_key.public_key() + private_numbers = private_key.private_numbers() + + self.encode_public(public_key, f_priv) + f_priv.put_mpint(private_numbers.private_value) + + +class _SSHFormatEd25519(object): + """Format for Ed25519 keys. + + Public: + bytes point + Private: + bytes point + bytes secret_and_point + """ + + def get_public(self, data): + """Ed25519 public fields""" + point, data = _get_sshstr(data) + return (point,), data + + def load_public(self, key_type, data, backend): + """Make Ed25519 public key from data.""" + (point,), data = self.get_public(data) + public_key = ed25519.Ed25519PublicKey.from_public_bytes( + point.tobytes() + ) + return public_key, data + + def load_private(self, data, pubfields, backend): + """Make Ed25519 private key from data.""" + (point,), data = self.get_public(data) + keypair, data = _get_sshstr(data) + + secret = keypair[:32] + point2 = keypair[32:] + if point != point2 or (point,) != pubfields: + raise ValueError("Corrupt data: ed25519 field mismatch") + private_key = ed25519.Ed25519PrivateKey.from_private_bytes(secret) + return private_key, data + + def encode_public(self, public_key, f_pub): + """Write Ed25519 public key""" + raw_public_key = public_key.public_bytes( + Encoding.Raw, PublicFormat.Raw + ) + f_pub.put_sshstr(raw_public_key) + + def encode_private(self, private_key, f_priv): + """Write Ed25519 private key""" + public_key = private_key.public_key() + raw_private_key = private_key.private_bytes( + Encoding.Raw, PrivateFormat.Raw, NoEncryption() + ) + raw_public_key = public_key.public_bytes( + Encoding.Raw, PublicFormat.Raw + ) + f_keypair = _FragList([raw_private_key, raw_public_key]) + + self.encode_public(public_key, f_priv) + f_priv.put_sshstr(f_keypair) + + +_KEY_FORMATS = { + _SSH_RSA: _SSHFormatRSA(), + _SSH_DSA: _SSHFormatDSA(), + _SSH_ED25519: _SSHFormatEd25519(), + _ECDSA_NISTP256: _SSHFormatECDSA(b"nistp256", ec.SECP256R1()), + _ECDSA_NISTP384: _SSHFormatECDSA(b"nistp384", ec.SECP384R1()), + _ECDSA_NISTP521: _SSHFormatECDSA(b"nistp521", ec.SECP521R1()), +} + + +def _lookup_kformat(key_type): + """Return valid format or throw error""" + if not isinstance(key_type, bytes): + key_type = memoryview(key_type).tobytes() + if key_type in _KEY_FORMATS: + return _KEY_FORMATS[key_type] + raise UnsupportedAlgorithm("Unsupported key type: %r" % key_type) + + +def load_ssh_private_key(data, password, backend=None): + """Load private key from OpenSSH custom encoding.""" + utils._check_byteslike("data", data) + backend = _get_backend(backend) + if password is not None: + utils._check_bytes("password", password) + + m = _PEM_RC.search(data) + if not m: + raise ValueError("Not OpenSSH private key format") + p1 = m.start(1) + p2 = m.end(1) + data = binascii.a2b_base64(memoryview(data)[p1:p2]) + if not data.startswith(_SK_MAGIC): + raise ValueError("Not OpenSSH private key format") + data = memoryview(data)[len(_SK_MAGIC) :] + + # parse header + ciphername, data = _get_sshstr(data) + kdfname, data = _get_sshstr(data) + kdfoptions, data = _get_sshstr(data) + nkeys, data = _get_u32(data) + if nkeys != 1: + raise ValueError("Only one key supported") + + # load public key data + pubdata, data = _get_sshstr(data) + pub_key_type, pubdata = _get_sshstr(pubdata) + kformat = _lookup_kformat(pub_key_type) + pubfields, pubdata = kformat.get_public(pubdata) + _check_empty(pubdata) + + # load secret data + edata, data = _get_sshstr(data) + _check_empty(data) + + if (ciphername, kdfname) != (_NONE, _NONE): + ciphername = ciphername.tobytes() + if ciphername not in _SSH_CIPHERS: + raise UnsupportedAlgorithm("Unsupported cipher: %r" % ciphername) + if kdfname != _BCRYPT: + raise UnsupportedAlgorithm("Unsupported KDF: %r" % kdfname) + blklen = _SSH_CIPHERS[ciphername][3] + _check_block_size(edata, blklen) + salt, kbuf = _get_sshstr(kdfoptions) + rounds, kbuf = _get_u32(kbuf) + _check_empty(kbuf) + ciph = _init_cipher( + ciphername, password, salt.tobytes(), rounds, backend + ) + edata = memoryview(ciph.decryptor().update(edata)) + else: + blklen = 8 + _check_block_size(edata, blklen) + ck1, edata = _get_u32(edata) + ck2, edata = _get_u32(edata) + if ck1 != ck2: + raise ValueError("Corrupt data: broken checksum") + + # load per-key struct + key_type, edata = _get_sshstr(edata) + if key_type != pub_key_type: + raise ValueError("Corrupt data: key type mismatch") + private_key, edata = kformat.load_private(edata, pubfields, backend) + comment, edata = _get_sshstr(edata) + + # yes, SSH does padding check *after* all other parsing is done. + # need to follow as it writes zero-byte padding too. + if edata != _PADDING[: len(edata)]: + raise ValueError("Corrupt data: invalid padding") + + return private_key + + +def serialize_ssh_private_key(private_key, password=None): + """Serialize private key with OpenSSH custom encoding.""" + if password is not None: + utils._check_bytes("password", password) + if password and len(password) > _MAX_PASSWORD: + raise ValueError( + "Passwords longer than 72 bytes are not supported by " + "OpenSSH private key format" + ) + + if isinstance(private_key, ec.EllipticCurvePrivateKey): + key_type = _ecdsa_key_type(private_key.public_key()) + elif isinstance(private_key, rsa.RSAPrivateKey): + key_type = _SSH_RSA + elif isinstance(private_key, dsa.DSAPrivateKey): + key_type = _SSH_DSA + elif isinstance(private_key, ed25519.Ed25519PrivateKey): + key_type = _SSH_ED25519 + else: + raise ValueError("Unsupported key type") + kformat = _lookup_kformat(key_type) + + # setup parameters + f_kdfoptions = _FragList() + if password: + ciphername = _DEFAULT_CIPHER + blklen = _SSH_CIPHERS[ciphername][3] + kdfname = _BCRYPT + rounds = _DEFAULT_ROUNDS + salt = os.urandom(16) + f_kdfoptions.put_sshstr(salt) + f_kdfoptions.put_u32(rounds) + backend = _get_backend(None) + ciph = _init_cipher(ciphername, password, salt, rounds, backend) + else: + ciphername = kdfname = _NONE + blklen = 8 + ciph = None + nkeys = 1 + checkval = os.urandom(4) + comment = b"" + + # encode public and private parts together + f_public_key = _FragList() + f_public_key.put_sshstr(key_type) + kformat.encode_public(private_key.public_key(), f_public_key) + + f_secrets = _FragList([checkval, checkval]) + f_secrets.put_sshstr(key_type) + kformat.encode_private(private_key, f_secrets) + f_secrets.put_sshstr(comment) + f_secrets.put_raw(_PADDING[: blklen - (f_secrets.size() % blklen)]) + + # top-level structure + f_main = _FragList() + f_main.put_raw(_SK_MAGIC) + f_main.put_sshstr(ciphername) + f_main.put_sshstr(kdfname) + f_main.put_sshstr(f_kdfoptions) + f_main.put_u32(nkeys) + f_main.put_sshstr(f_public_key) + f_main.put_sshstr(f_secrets) + + # copy result info bytearray + slen = f_secrets.size() + mlen = f_main.size() + buf = memoryview(bytearray(mlen + blklen)) + f_main.render(buf) + ofs = mlen - slen + + # encrypt in-place + if ciph is not None: + ciph.encryptor().update_into(buf[ofs:mlen], buf[ofs:]) + + txt = _ssh_pem_encode(buf[:mlen]) + buf[ofs:mlen] = bytearray(slen) + return txt + + +def load_ssh_public_key(data, backend=None): + """Load public key from OpenSSH one-line format.""" + backend = _get_backend(backend) + utils._check_byteslike("data", data) + + m = _SSH_PUBKEY_RC.match(data) + if not m: + raise ValueError("Invalid line format") + key_type = orig_key_type = m.group(1) + key_body = m.group(2) + with_cert = False + if _CERT_SUFFIX == key_type[-len(_CERT_SUFFIX) :]: + with_cert = True + key_type = key_type[: -len(_CERT_SUFFIX)] + kformat = _lookup_kformat(key_type) + + try: + data = memoryview(binascii.a2b_base64(key_body)) + except (TypeError, binascii.Error): + raise ValueError("Invalid key format") + + inner_key_type, data = _get_sshstr(data) + if inner_key_type != orig_key_type: + raise ValueError("Invalid key format") + if with_cert: + nonce, data = _get_sshstr(data) + public_key, data = kformat.load_public(key_type, data, backend) + if with_cert: + serial, data = _get_u64(data) + cctype, data = _get_u32(data) + key_id, data = _get_sshstr(data) + principals, data = _get_sshstr(data) + valid_after, data = _get_u64(data) + valid_before, data = _get_u64(data) + crit_options, data = _get_sshstr(data) + extensions, data = _get_sshstr(data) + reserved, data = _get_sshstr(data) + sig_key, data = _get_sshstr(data) + signature, data = _get_sshstr(data) + _check_empty(data) + return public_key + + +def serialize_ssh_public_key(public_key): + """One-line public key format for OpenSSH""" + if isinstance(public_key, ec.EllipticCurvePublicKey): + key_type = _ecdsa_key_type(public_key) + elif isinstance(public_key, rsa.RSAPublicKey): + key_type = _SSH_RSA + elif isinstance(public_key, dsa.DSAPublicKey): + key_type = _SSH_DSA + elif isinstance(public_key, ed25519.Ed25519PublicKey): + key_type = _SSH_ED25519 + else: + raise ValueError("Unsupported key type") + kformat = _lookup_kformat(key_type) + + f_pub = _FragList() + f_pub.put_sshstr(key_type) + kformat.encode_public(public_key, f_pub) + + pub = binascii.b2a_base64(f_pub.tobytes()).strip() + return b"".join([key_type, b" ", pub]) diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/twofactor/__init__.py b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/twofactor/__init__.py new file mode 100644 index 0000000..e71f9e6 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/twofactor/__init__.py @@ -0,0 +1,9 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function + + +class InvalidToken(Exception): + pass diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/twofactor/__pycache__/__init__.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/twofactor/__pycache__/__init__.cpython-39.pyc new file mode 100644 index 0000000..fe83afc Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/twofactor/__pycache__/__init__.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/twofactor/__pycache__/hotp.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/twofactor/__pycache__/hotp.cpython-39.pyc new file mode 100644 index 0000000..b22a7fb Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/twofactor/__pycache__/hotp.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/twofactor/__pycache__/totp.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/twofactor/__pycache__/totp.cpython-39.pyc new file mode 100644 index 0000000..f855f96 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/twofactor/__pycache__/totp.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/twofactor/__pycache__/utils.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/twofactor/__pycache__/utils.cpython-39.pyc new file mode 100644 index 0000000..915e29c Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/twofactor/__pycache__/utils.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/twofactor/hotp.py b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/twofactor/hotp.py new file mode 100644 index 0000000..c00eec0 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/twofactor/hotp.py @@ -0,0 +1,69 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function + +import struct + +import six + +from cryptography.exceptions import UnsupportedAlgorithm, _Reasons +from cryptography.hazmat.backends import _get_backend +from cryptography.hazmat.backends.interfaces import HMACBackend +from cryptography.hazmat.primitives import constant_time, hmac +from cryptography.hazmat.primitives.hashes import SHA1, SHA256, SHA512 +from cryptography.hazmat.primitives.twofactor import InvalidToken +from cryptography.hazmat.primitives.twofactor.utils import _generate_uri + + +class HOTP(object): + def __init__( + self, key, length, algorithm, backend=None, enforce_key_length=True + ): + backend = _get_backend(backend) + if not isinstance(backend, HMACBackend): + raise UnsupportedAlgorithm( + "Backend object does not implement HMACBackend.", + _Reasons.BACKEND_MISSING_INTERFACE, + ) + + if len(key) < 16 and enforce_key_length is True: + raise ValueError("Key length has to be at least 128 bits.") + + if not isinstance(length, six.integer_types): + raise TypeError("Length parameter must be an integer type.") + + if length < 6 or length > 8: + raise ValueError("Length of HOTP has to be between 6 to 8.") + + if not isinstance(algorithm, (SHA1, SHA256, SHA512)): + raise TypeError("Algorithm must be SHA1, SHA256 or SHA512.") + + self._key = key + self._length = length + self._algorithm = algorithm + self._backend = backend + + def generate(self, counter): + truncated_value = self._dynamic_truncate(counter) + hotp = truncated_value % (10 ** self._length) + return "{0:0{1}}".format(hotp, self._length).encode() + + def verify(self, hotp, counter): + if not constant_time.bytes_eq(self.generate(counter), hotp): + raise InvalidToken("Supplied HOTP value does not match.") + + def _dynamic_truncate(self, counter): + ctx = hmac.HMAC(self._key, self._algorithm, self._backend) + ctx.update(struct.pack(">Q", counter)) + hmac_value = ctx.finalize() + + offset = six.indexbytes(hmac_value, len(hmac_value) - 1) & 0b1111 + p = hmac_value[offset : offset + 4] + return struct.unpack(">I", p)[0] & 0x7FFFFFFF + + def get_provisioning_uri(self, account_name, counter, issuer): + return _generate_uri( + self, "hotp", account_name, issuer, [("counter", int(counter))] + ) diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/twofactor/totp.py b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/twofactor/totp.py new file mode 100644 index 0000000..d59539b --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/twofactor/totp.py @@ -0,0 +1,51 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function + +from cryptography.exceptions import UnsupportedAlgorithm, _Reasons +from cryptography.hazmat.backends import _get_backend +from cryptography.hazmat.backends.interfaces import HMACBackend +from cryptography.hazmat.primitives import constant_time +from cryptography.hazmat.primitives.twofactor import InvalidToken +from cryptography.hazmat.primitives.twofactor.hotp import HOTP +from cryptography.hazmat.primitives.twofactor.utils import _generate_uri + + +class TOTP(object): + def __init__( + self, + key, + length, + algorithm, + time_step, + backend=None, + enforce_key_length=True, + ): + backend = _get_backend(backend) + if not isinstance(backend, HMACBackend): + raise UnsupportedAlgorithm( + "Backend object does not implement HMACBackend.", + _Reasons.BACKEND_MISSING_INTERFACE, + ) + + self._time_step = time_step + self._hotp = HOTP(key, length, algorithm, backend, enforce_key_length) + + def generate(self, time): + counter = int(time / self._time_step) + return self._hotp.generate(counter) + + def verify(self, totp, time): + if not constant_time.bytes_eq(self.generate(time), totp): + raise InvalidToken("Supplied TOTP value does not match.") + + def get_provisioning_uri(self, account_name, issuer): + return _generate_uri( + self._hotp, + "totp", + account_name, + issuer, + [("period", int(self._time_step))], + ) diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/twofactor/utils.py b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/twofactor/utils.py new file mode 100644 index 0000000..0afa1cc --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/cryptography/hazmat/primitives/twofactor/utils.py @@ -0,0 +1,33 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function + +import base64 + +from six.moves.urllib.parse import quote, urlencode + + +def _generate_uri(hotp, type_name, account_name, issuer, extra_parameters): + parameters = [ + ("digits", hotp._length), + ("secret", base64.b32encode(hotp._key)), + ("algorithm", hotp._algorithm.name.upper()), + ] + + if issuer is not None: + parameters.append(("issuer", issuer)) + + parameters.extend(extra_parameters) + + uriparts = { + "type": type_name, + "label": ( + "%s:%s" % (quote(issuer), quote(account_name)) + if issuer + else quote(account_name) + ), + "parameters": urlencode(parameters), + } + return "otpauth://{type}/{label}?{parameters}".format(**uriparts) diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/utils.py b/WebCrawler/venv/Lib/site-packages/cryptography/utils.py new file mode 100644 index 0000000..bdb3dbf --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/cryptography/utils.py @@ -0,0 +1,171 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function + +import abc +import binascii +import inspect +import sys +import warnings + + +# We use a UserWarning subclass, instead of DeprecationWarning, because CPython +# decided deprecation warnings should be invisble by default. +class CryptographyDeprecationWarning(UserWarning): + pass + + +# Several APIs were deprecated with no specific end-of-life date because of the +# ubiquity of their use. They should not be removed until we agree on when that +# cycle ends. +PersistentlyDeprecated2017 = CryptographyDeprecationWarning +PersistentlyDeprecated2019 = CryptographyDeprecationWarning + + +def _check_bytes(name, value): + if not isinstance(value, bytes): + raise TypeError("{} must be bytes".format(name)) + + +def _check_byteslike(name, value): + try: + memoryview(value) + except TypeError: + raise TypeError("{} must be bytes-like".format(name)) + + +def read_only_property(name): + return property(lambda self: getattr(self, name)) + + +def register_interface(iface): + def register_decorator(klass): + verify_interface(iface, klass) + iface.register(klass) + return klass + + return register_decorator + + +def register_interface_if(predicate, iface): + def register_decorator(klass): + if predicate: + verify_interface(iface, klass) + iface.register(klass) + return klass + + return register_decorator + + +if hasattr(int, "from_bytes"): + int_from_bytes = int.from_bytes +else: + + def int_from_bytes(data, byteorder, signed=False): + assert byteorder == "big" + assert not signed + + return int(binascii.hexlify(data), 16) + + +if hasattr(int, "to_bytes"): + + def int_to_bytes(integer, length=None): + return integer.to_bytes( + length or (integer.bit_length() + 7) // 8 or 1, "big" + ) + + +else: + + def int_to_bytes(integer, length=None): + hex_string = "%x" % integer + if length is None: + n = len(hex_string) + else: + n = length * 2 + return binascii.unhexlify(hex_string.zfill(n + (n & 1))) + + +class InterfaceNotImplemented(Exception): + pass + + +if hasattr(inspect, "signature"): + signature = inspect.signature +else: + signature = inspect.getargspec + + +def verify_interface(iface, klass): + for method in iface.__abstractmethods__: + if not hasattr(klass, method): + raise InterfaceNotImplemented( + "{} is missing a {!r} method".format(klass, method) + ) + if isinstance(getattr(iface, method), abc.abstractproperty): + # Can't properly verify these yet. + continue + sig = signature(getattr(iface, method)) + actual = signature(getattr(klass, method)) + if sig != actual: + raise InterfaceNotImplemented( + "{}.{}'s signature differs from the expected. Expected: " + "{!r}. Received: {!r}".format(klass, method, sig, actual) + ) + + +class _DeprecatedValue(object): + def __init__(self, value, message, warning_class): + self.value = value + self.message = message + self.warning_class = warning_class + + +class _ModuleWithDeprecations(object): + def __init__(self, module): + self.__dict__["_module"] = module + + def __getattr__(self, attr): + obj = getattr(self._module, attr) + if isinstance(obj, _DeprecatedValue): + warnings.warn(obj.message, obj.warning_class, stacklevel=2) + obj = obj.value + return obj + + def __setattr__(self, attr, value): + setattr(self._module, attr, value) + + def __delattr__(self, attr): + obj = getattr(self._module, attr) + if isinstance(obj, _DeprecatedValue): + warnings.warn(obj.message, obj.warning_class, stacklevel=2) + + delattr(self._module, attr) + + def __dir__(self): + return ["_module"] + dir(self._module) + + +def deprecated(value, module_name, message, warning_class): + module = sys.modules[module_name] + if not isinstance(module, _ModuleWithDeprecations): + sys.modules[module_name] = _ModuleWithDeprecations(module) + return _DeprecatedValue(value, message, warning_class) + + +def cached_property(func): + cached_name = "_cached_{}".format(func) + sentinel = object() + + def inner(instance): + cache = getattr(instance, cached_name, sentinel) + if cache is not sentinel: + return cache + result = func(instance) + setattr(instance, cached_name, result) + return result + + return property(inner) diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/x509/__init__.py b/WebCrawler/venv/Lib/site-packages/cryptography/x509/__init__.py new file mode 100644 index 0000000..69630e4 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/cryptography/x509/__init__.py @@ -0,0 +1,248 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function + +from cryptography.x509 import certificate_transparency +from cryptography.x509.base import ( + AttributeNotFound, + Certificate, + CertificateBuilder, + CertificateRevocationList, + CertificateRevocationListBuilder, + CertificateSigningRequest, + CertificateSigningRequestBuilder, + InvalidVersion, + RevokedCertificate, + RevokedCertificateBuilder, + Version, + load_der_x509_certificate, + load_der_x509_crl, + load_der_x509_csr, + load_pem_x509_certificate, + load_pem_x509_crl, + load_pem_x509_csr, + random_serial_number, +) +from cryptography.x509.extensions import ( + AccessDescription, + AuthorityInformationAccess, + AuthorityKeyIdentifier, + BasicConstraints, + CRLDistributionPoints, + CRLNumber, + CRLReason, + CertificateIssuer, + CertificatePolicies, + DeltaCRLIndicator, + DistributionPoint, + DuplicateExtension, + ExtendedKeyUsage, + Extension, + ExtensionNotFound, + ExtensionType, + Extensions, + FreshestCRL, + GeneralNames, + InhibitAnyPolicy, + InvalidityDate, + IssuerAlternativeName, + IssuingDistributionPoint, + KeyUsage, + NameConstraints, + NoticeReference, + OCSPNoCheck, + OCSPNonce, + PolicyConstraints, + PolicyInformation, + PrecertPoison, + PrecertificateSignedCertificateTimestamps, + ReasonFlags, + SignedCertificateTimestamps, + SubjectAlternativeName, + SubjectInformationAccess, + SubjectKeyIdentifier, + TLSFeature, + TLSFeatureType, + UnrecognizedExtension, + UserNotice, +) +from cryptography.x509.general_name import ( + DNSName, + DirectoryName, + GeneralName, + IPAddress, + OtherName, + RFC822Name, + RegisteredID, + UniformResourceIdentifier, + UnsupportedGeneralNameType, + _GENERAL_NAMES, +) +from cryptography.x509.name import ( + Name, + NameAttribute, + RelativeDistinguishedName, +) +from cryptography.x509.oid import ( + AuthorityInformationAccessOID, + CRLEntryExtensionOID, + CertificatePoliciesOID, + ExtendedKeyUsageOID, + ExtensionOID, + NameOID, + ObjectIdentifier, + SignatureAlgorithmOID, + _SIG_OIDS_TO_HASH, +) + + +OID_AUTHORITY_INFORMATION_ACCESS = ExtensionOID.AUTHORITY_INFORMATION_ACCESS +OID_AUTHORITY_KEY_IDENTIFIER = ExtensionOID.AUTHORITY_KEY_IDENTIFIER +OID_BASIC_CONSTRAINTS = ExtensionOID.BASIC_CONSTRAINTS +OID_CERTIFICATE_POLICIES = ExtensionOID.CERTIFICATE_POLICIES +OID_CRL_DISTRIBUTION_POINTS = ExtensionOID.CRL_DISTRIBUTION_POINTS +OID_EXTENDED_KEY_USAGE = ExtensionOID.EXTENDED_KEY_USAGE +OID_FRESHEST_CRL = ExtensionOID.FRESHEST_CRL +OID_INHIBIT_ANY_POLICY = ExtensionOID.INHIBIT_ANY_POLICY +OID_ISSUER_ALTERNATIVE_NAME = ExtensionOID.ISSUER_ALTERNATIVE_NAME +OID_KEY_USAGE = ExtensionOID.KEY_USAGE +OID_NAME_CONSTRAINTS = ExtensionOID.NAME_CONSTRAINTS +OID_OCSP_NO_CHECK = ExtensionOID.OCSP_NO_CHECK +OID_POLICY_CONSTRAINTS = ExtensionOID.POLICY_CONSTRAINTS +OID_POLICY_MAPPINGS = ExtensionOID.POLICY_MAPPINGS +OID_SUBJECT_ALTERNATIVE_NAME = ExtensionOID.SUBJECT_ALTERNATIVE_NAME +OID_SUBJECT_DIRECTORY_ATTRIBUTES = ExtensionOID.SUBJECT_DIRECTORY_ATTRIBUTES +OID_SUBJECT_INFORMATION_ACCESS = ExtensionOID.SUBJECT_INFORMATION_ACCESS +OID_SUBJECT_KEY_IDENTIFIER = ExtensionOID.SUBJECT_KEY_IDENTIFIER + +OID_DSA_WITH_SHA1 = SignatureAlgorithmOID.DSA_WITH_SHA1 +OID_DSA_WITH_SHA224 = SignatureAlgorithmOID.DSA_WITH_SHA224 +OID_DSA_WITH_SHA256 = SignatureAlgorithmOID.DSA_WITH_SHA256 +OID_ECDSA_WITH_SHA1 = SignatureAlgorithmOID.ECDSA_WITH_SHA1 +OID_ECDSA_WITH_SHA224 = SignatureAlgorithmOID.ECDSA_WITH_SHA224 +OID_ECDSA_WITH_SHA256 = SignatureAlgorithmOID.ECDSA_WITH_SHA256 +OID_ECDSA_WITH_SHA384 = SignatureAlgorithmOID.ECDSA_WITH_SHA384 +OID_ECDSA_WITH_SHA512 = SignatureAlgorithmOID.ECDSA_WITH_SHA512 +OID_RSA_WITH_MD5 = SignatureAlgorithmOID.RSA_WITH_MD5 +OID_RSA_WITH_SHA1 = SignatureAlgorithmOID.RSA_WITH_SHA1 +OID_RSA_WITH_SHA224 = SignatureAlgorithmOID.RSA_WITH_SHA224 +OID_RSA_WITH_SHA256 = SignatureAlgorithmOID.RSA_WITH_SHA256 +OID_RSA_WITH_SHA384 = SignatureAlgorithmOID.RSA_WITH_SHA384 +OID_RSA_WITH_SHA512 = SignatureAlgorithmOID.RSA_WITH_SHA512 +OID_RSASSA_PSS = SignatureAlgorithmOID.RSASSA_PSS + +OID_COMMON_NAME = NameOID.COMMON_NAME +OID_COUNTRY_NAME = NameOID.COUNTRY_NAME +OID_DOMAIN_COMPONENT = NameOID.DOMAIN_COMPONENT +OID_DN_QUALIFIER = NameOID.DN_QUALIFIER +OID_EMAIL_ADDRESS = NameOID.EMAIL_ADDRESS +OID_GENERATION_QUALIFIER = NameOID.GENERATION_QUALIFIER +OID_GIVEN_NAME = NameOID.GIVEN_NAME +OID_LOCALITY_NAME = NameOID.LOCALITY_NAME +OID_ORGANIZATIONAL_UNIT_NAME = NameOID.ORGANIZATIONAL_UNIT_NAME +OID_ORGANIZATION_NAME = NameOID.ORGANIZATION_NAME +OID_PSEUDONYM = NameOID.PSEUDONYM +OID_SERIAL_NUMBER = NameOID.SERIAL_NUMBER +OID_STATE_OR_PROVINCE_NAME = NameOID.STATE_OR_PROVINCE_NAME +OID_SURNAME = NameOID.SURNAME +OID_TITLE = NameOID.TITLE + +OID_CLIENT_AUTH = ExtendedKeyUsageOID.CLIENT_AUTH +OID_CODE_SIGNING = ExtendedKeyUsageOID.CODE_SIGNING +OID_EMAIL_PROTECTION = ExtendedKeyUsageOID.EMAIL_PROTECTION +OID_OCSP_SIGNING = ExtendedKeyUsageOID.OCSP_SIGNING +OID_SERVER_AUTH = ExtendedKeyUsageOID.SERVER_AUTH +OID_TIME_STAMPING = ExtendedKeyUsageOID.TIME_STAMPING + +OID_ANY_POLICY = CertificatePoliciesOID.ANY_POLICY +OID_CPS_QUALIFIER = CertificatePoliciesOID.CPS_QUALIFIER +OID_CPS_USER_NOTICE = CertificatePoliciesOID.CPS_USER_NOTICE + +OID_CERTIFICATE_ISSUER = CRLEntryExtensionOID.CERTIFICATE_ISSUER +OID_CRL_REASON = CRLEntryExtensionOID.CRL_REASON +OID_INVALIDITY_DATE = CRLEntryExtensionOID.INVALIDITY_DATE + +OID_CA_ISSUERS = AuthorityInformationAccessOID.CA_ISSUERS +OID_OCSP = AuthorityInformationAccessOID.OCSP + +__all__ = [ + "certificate_transparency", + "load_pem_x509_certificate", + "load_der_x509_certificate", + "load_pem_x509_csr", + "load_der_x509_csr", + "load_pem_x509_crl", + "load_der_x509_crl", + "random_serial_number", + "AttributeNotFound", + "InvalidVersion", + "DeltaCRLIndicator", + "DuplicateExtension", + "ExtensionNotFound", + "UnsupportedGeneralNameType", + "NameAttribute", + "Name", + "RelativeDistinguishedName", + "ObjectIdentifier", + "ExtensionType", + "Extensions", + "Extension", + "ExtendedKeyUsage", + "FreshestCRL", + "IssuingDistributionPoint", + "TLSFeature", + "TLSFeatureType", + "OCSPNoCheck", + "BasicConstraints", + "CRLNumber", + "KeyUsage", + "AuthorityInformationAccess", + "SubjectInformationAccess", + "AccessDescription", + "CertificatePolicies", + "PolicyInformation", + "UserNotice", + "NoticeReference", + "SubjectKeyIdentifier", + "NameConstraints", + "CRLDistributionPoints", + "DistributionPoint", + "ReasonFlags", + "InhibitAnyPolicy", + "SubjectAlternativeName", + "IssuerAlternativeName", + "AuthorityKeyIdentifier", + "GeneralNames", + "GeneralName", + "RFC822Name", + "DNSName", + "UniformResourceIdentifier", + "RegisteredID", + "DirectoryName", + "IPAddress", + "OtherName", + "Certificate", + "CertificateRevocationList", + "CertificateRevocationListBuilder", + "CertificateSigningRequest", + "RevokedCertificate", + "RevokedCertificateBuilder", + "CertificateSigningRequestBuilder", + "CertificateBuilder", + "Version", + "_SIG_OIDS_TO_HASH", + "OID_CA_ISSUERS", + "OID_OCSP", + "_GENERAL_NAMES", + "CertificateIssuer", + "CRLReason", + "InvalidityDate", + "UnrecognizedExtension", + "PolicyConstraints", + "PrecertificateSignedCertificateTimestamps", + "PrecertPoison", + "OCSPNonce", + "SignedCertificateTimestamps", +] diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/x509/__pycache__/__init__.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/cryptography/x509/__pycache__/__init__.cpython-39.pyc new file mode 100644 index 0000000..6a3964b Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/cryptography/x509/__pycache__/__init__.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/x509/__pycache__/base.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/cryptography/x509/__pycache__/base.cpython-39.pyc new file mode 100644 index 0000000..da49dc7 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/cryptography/x509/__pycache__/base.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/x509/__pycache__/certificate_transparency.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/cryptography/x509/__pycache__/certificate_transparency.cpython-39.pyc new file mode 100644 index 0000000..9c19f1e Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/cryptography/x509/__pycache__/certificate_transparency.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/x509/__pycache__/extensions.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/cryptography/x509/__pycache__/extensions.cpython-39.pyc new file mode 100644 index 0000000..794b63f Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/cryptography/x509/__pycache__/extensions.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/x509/__pycache__/general_name.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/cryptography/x509/__pycache__/general_name.cpython-39.pyc new file mode 100644 index 0000000..5569982 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/cryptography/x509/__pycache__/general_name.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/x509/__pycache__/name.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/cryptography/x509/__pycache__/name.cpython-39.pyc new file mode 100644 index 0000000..696d35d Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/cryptography/x509/__pycache__/name.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/x509/__pycache__/ocsp.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/cryptography/x509/__pycache__/ocsp.cpython-39.pyc new file mode 100644 index 0000000..e272260 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/cryptography/x509/__pycache__/ocsp.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/x509/__pycache__/oid.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/cryptography/x509/__pycache__/oid.cpython-39.pyc new file mode 100644 index 0000000..96b6ad6 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/cryptography/x509/__pycache__/oid.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/x509/base.py b/WebCrawler/venv/Lib/site-packages/cryptography/x509/base.py new file mode 100644 index 0000000..f3bc872 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/cryptography/x509/base.py @@ -0,0 +1,892 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function + +import abc +import datetime +import os +from enum import Enum + +import six + +from cryptography import utils +from cryptography.hazmat.backends import _get_backend +from cryptography.hazmat.primitives.asymmetric import ( + dsa, + ec, + ed25519, + ed448, + rsa, +) +from cryptography.x509.extensions import Extension, ExtensionType +from cryptography.x509.name import Name +from cryptography.x509.oid import ObjectIdentifier + + +_EARLIEST_UTC_TIME = datetime.datetime(1950, 1, 1) + + +class AttributeNotFound(Exception): + def __init__(self, msg, oid): + super(AttributeNotFound, self).__init__(msg) + self.oid = oid + + +def _reject_duplicate_extension(extension, extensions): + # This is quadratic in the number of extensions + for e in extensions: + if e.oid == extension.oid: + raise ValueError("This extension has already been set.") + + +def _reject_duplicate_attribute(oid, attributes): + # This is quadratic in the number of attributes + for attr_oid, _ in attributes: + if attr_oid == oid: + raise ValueError("This attribute has already been set.") + + +def _convert_to_naive_utc_time(time): + """Normalizes a datetime to a naive datetime in UTC. + + time -- datetime to normalize. Assumed to be in UTC if not timezone + aware. + """ + if time.tzinfo is not None: + offset = time.utcoffset() + offset = offset if offset else datetime.timedelta() + return time.replace(tzinfo=None) - offset + else: + return time + + +class Version(Enum): + v1 = 0 + v3 = 2 + + +def load_pem_x509_certificate(data, backend=None): + backend = _get_backend(backend) + return backend.load_pem_x509_certificate(data) + + +def load_der_x509_certificate(data, backend=None): + backend = _get_backend(backend) + return backend.load_der_x509_certificate(data) + + +def load_pem_x509_csr(data, backend=None): + backend = _get_backend(backend) + return backend.load_pem_x509_csr(data) + + +def load_der_x509_csr(data, backend=None): + backend = _get_backend(backend) + return backend.load_der_x509_csr(data) + + +def load_pem_x509_crl(data, backend=None): + backend = _get_backend(backend) + return backend.load_pem_x509_crl(data) + + +def load_der_x509_crl(data, backend=None): + backend = _get_backend(backend) + return backend.load_der_x509_crl(data) + + +class InvalidVersion(Exception): + def __init__(self, msg, parsed_version): + super(InvalidVersion, self).__init__(msg) + self.parsed_version = parsed_version + + +@six.add_metaclass(abc.ABCMeta) +class Certificate(object): + @abc.abstractmethod + def fingerprint(self, algorithm): + """ + Returns bytes using digest passed. + """ + + @abc.abstractproperty + def serial_number(self): + """ + Returns certificate serial number + """ + + @abc.abstractproperty + def version(self): + """ + Returns the certificate version + """ + + @abc.abstractmethod + def public_key(self): + """ + Returns the public key + """ + + @abc.abstractproperty + def not_valid_before(self): + """ + Not before time (represented as UTC datetime) + """ + + @abc.abstractproperty + def not_valid_after(self): + """ + Not after time (represented as UTC datetime) + """ + + @abc.abstractproperty + def issuer(self): + """ + Returns the issuer name object. + """ + + @abc.abstractproperty + def subject(self): + """ + Returns the subject name object. + """ + + @abc.abstractproperty + def signature_hash_algorithm(self): + """ + Returns a HashAlgorithm corresponding to the type of the digest signed + in the certificate. + """ + + @abc.abstractproperty + def signature_algorithm_oid(self): + """ + Returns the ObjectIdentifier of the signature algorithm. + """ + + @abc.abstractproperty + def extensions(self): + """ + Returns an Extensions object. + """ + + @abc.abstractproperty + def signature(self): + """ + Returns the signature bytes. + """ + + @abc.abstractproperty + def tbs_certificate_bytes(self): + """ + Returns the tbsCertificate payload bytes as defined in RFC 5280. + """ + + @abc.abstractmethod + def __eq__(self, other): + """ + Checks equality. + """ + + @abc.abstractmethod + def __ne__(self, other): + """ + Checks not equal. + """ + + @abc.abstractmethod + def __hash__(self): + """ + Computes a hash. + """ + + @abc.abstractmethod + def public_bytes(self, encoding): + """ + Serializes the certificate to PEM or DER format. + """ + + +@six.add_metaclass(abc.ABCMeta) +class CertificateRevocationList(object): + @abc.abstractmethod + def public_bytes(self, encoding): + """ + Serializes the CRL to PEM or DER format. + """ + + @abc.abstractmethod + def fingerprint(self, algorithm): + """ + Returns bytes using digest passed. + """ + + @abc.abstractmethod + def get_revoked_certificate_by_serial_number(self, serial_number): + """ + Returns an instance of RevokedCertificate or None if the serial_number + is not in the CRL. + """ + + @abc.abstractproperty + def signature_hash_algorithm(self): + """ + Returns a HashAlgorithm corresponding to the type of the digest signed + in the certificate. + """ + + @abc.abstractproperty + def signature_algorithm_oid(self): + """ + Returns the ObjectIdentifier of the signature algorithm. + """ + + @abc.abstractproperty + def issuer(self): + """ + Returns the X509Name with the issuer of this CRL. + """ + + @abc.abstractproperty + def next_update(self): + """ + Returns the date of next update for this CRL. + """ + + @abc.abstractproperty + def last_update(self): + """ + Returns the date of last update for this CRL. + """ + + @abc.abstractproperty + def extensions(self): + """ + Returns an Extensions object containing a list of CRL extensions. + """ + + @abc.abstractproperty + def signature(self): + """ + Returns the signature bytes. + """ + + @abc.abstractproperty + def tbs_certlist_bytes(self): + """ + Returns the tbsCertList payload bytes as defined in RFC 5280. + """ + + @abc.abstractmethod + def __eq__(self, other): + """ + Checks equality. + """ + + @abc.abstractmethod + def __ne__(self, other): + """ + Checks not equal. + """ + + @abc.abstractmethod + def __len__(self): + """ + Number of revoked certificates in the CRL. + """ + + @abc.abstractmethod + def __getitem__(self, idx): + """ + Returns a revoked certificate (or slice of revoked certificates). + """ + + @abc.abstractmethod + def __iter__(self): + """ + Iterator over the revoked certificates + """ + + @abc.abstractmethod + def is_signature_valid(self, public_key): + """ + Verifies signature of revocation list against given public key. + """ + + +@six.add_metaclass(abc.ABCMeta) +class CertificateSigningRequest(object): + @abc.abstractmethod + def __eq__(self, other): + """ + Checks equality. + """ + + @abc.abstractmethod + def __ne__(self, other): + """ + Checks not equal. + """ + + @abc.abstractmethod + def __hash__(self): + """ + Computes a hash. + """ + + @abc.abstractmethod + def public_key(self): + """ + Returns the public key + """ + + @abc.abstractproperty + def subject(self): + """ + Returns the subject name object. + """ + + @abc.abstractproperty + def signature_hash_algorithm(self): + """ + Returns a HashAlgorithm corresponding to the type of the digest signed + in the certificate. + """ + + @abc.abstractproperty + def signature_algorithm_oid(self): + """ + Returns the ObjectIdentifier of the signature algorithm. + """ + + @abc.abstractproperty + def extensions(self): + """ + Returns the extensions in the signing request. + """ + + @abc.abstractmethod + def public_bytes(self, encoding): + """ + Encodes the request to PEM or DER format. + """ + + @abc.abstractproperty + def signature(self): + """ + Returns the signature bytes. + """ + + @abc.abstractproperty + def tbs_certrequest_bytes(self): + """ + Returns the PKCS#10 CertificationRequestInfo bytes as defined in RFC + 2986. + """ + + @abc.abstractproperty + def is_signature_valid(self): + """ + Verifies signature of signing request. + """ + + @abc.abstractproperty + def get_attribute_for_oid(self): + """ + Get the attribute value for a given OID. + """ + + +@six.add_metaclass(abc.ABCMeta) +class RevokedCertificate(object): + @abc.abstractproperty + def serial_number(self): + """ + Returns the serial number of the revoked certificate. + """ + + @abc.abstractproperty + def revocation_date(self): + """ + Returns the date of when this certificate was revoked. + """ + + @abc.abstractproperty + def extensions(self): + """ + Returns an Extensions object containing a list of Revoked extensions. + """ + + +class CertificateSigningRequestBuilder(object): + def __init__(self, subject_name=None, extensions=[], attributes=[]): + """ + Creates an empty X.509 certificate request (v1). + """ + self._subject_name = subject_name + self._extensions = extensions + self._attributes = attributes + + def subject_name(self, name): + """ + Sets the certificate requestor's distinguished name. + """ + if not isinstance(name, Name): + raise TypeError("Expecting x509.Name object.") + if self._subject_name is not None: + raise ValueError("The subject name may only be set once.") + return CertificateSigningRequestBuilder( + name, self._extensions, self._attributes + ) + + def add_extension(self, extension, critical): + """ + Adds an X.509 extension to the certificate request. + """ + if not isinstance(extension, ExtensionType): + raise TypeError("extension must be an ExtensionType") + + extension = Extension(extension.oid, critical, extension) + _reject_duplicate_extension(extension, self._extensions) + + return CertificateSigningRequestBuilder( + self._subject_name, + self._extensions + [extension], + self._attributes, + ) + + def add_attribute(self, oid, value): + """ + Adds an X.509 attribute with an OID and associated value. + """ + if not isinstance(oid, ObjectIdentifier): + raise TypeError("oid must be an ObjectIdentifier") + + if not isinstance(value, bytes): + raise TypeError("value must be bytes") + + _reject_duplicate_attribute(oid, self._attributes) + + return CertificateSigningRequestBuilder( + self._subject_name, + self._extensions, + self._attributes + [(oid, value)], + ) + + def sign(self, private_key, algorithm, backend=None): + """ + Signs the request using the requestor's private key. + """ + backend = _get_backend(backend) + if self._subject_name is None: + raise ValueError("A CertificateSigningRequest must have a subject") + return backend.create_x509_csr(self, private_key, algorithm) + + +class CertificateBuilder(object): + def __init__( + self, + issuer_name=None, + subject_name=None, + public_key=None, + serial_number=None, + not_valid_before=None, + not_valid_after=None, + extensions=[], + ): + self._version = Version.v3 + self._issuer_name = issuer_name + self._subject_name = subject_name + self._public_key = public_key + self._serial_number = serial_number + self._not_valid_before = not_valid_before + self._not_valid_after = not_valid_after + self._extensions = extensions + + def issuer_name(self, name): + """ + Sets the CA's distinguished name. + """ + if not isinstance(name, Name): + raise TypeError("Expecting x509.Name object.") + if self._issuer_name is not None: + raise ValueError("The issuer name may only be set once.") + return CertificateBuilder( + name, + self._subject_name, + self._public_key, + self._serial_number, + self._not_valid_before, + self._not_valid_after, + self._extensions, + ) + + def subject_name(self, name): + """ + Sets the requestor's distinguished name. + """ + if not isinstance(name, Name): + raise TypeError("Expecting x509.Name object.") + if self._subject_name is not None: + raise ValueError("The subject name may only be set once.") + return CertificateBuilder( + self._issuer_name, + name, + self._public_key, + self._serial_number, + self._not_valid_before, + self._not_valid_after, + self._extensions, + ) + + def public_key(self, key): + """ + Sets the requestor's public key (as found in the signing request). + """ + if not isinstance( + key, + ( + dsa.DSAPublicKey, + rsa.RSAPublicKey, + ec.EllipticCurvePublicKey, + ed25519.Ed25519PublicKey, + ed448.Ed448PublicKey, + ), + ): + raise TypeError( + "Expecting one of DSAPublicKey, RSAPublicKey," + " EllipticCurvePublicKey, Ed25519PublicKey or" + " Ed448PublicKey." + ) + if self._public_key is not None: + raise ValueError("The public key may only be set once.") + return CertificateBuilder( + self._issuer_name, + self._subject_name, + key, + self._serial_number, + self._not_valid_before, + self._not_valid_after, + self._extensions, + ) + + def serial_number(self, number): + """ + Sets the certificate serial number. + """ + if not isinstance(number, six.integer_types): + raise TypeError("Serial number must be of integral type.") + if self._serial_number is not None: + raise ValueError("The serial number may only be set once.") + if number <= 0: + raise ValueError("The serial number should be positive.") + + # ASN.1 integers are always signed, so most significant bit must be + # zero. + if number.bit_length() >= 160: # As defined in RFC 5280 + raise ValueError( + "The serial number should not be more than 159 " "bits." + ) + return CertificateBuilder( + self._issuer_name, + self._subject_name, + self._public_key, + number, + self._not_valid_before, + self._not_valid_after, + self._extensions, + ) + + def not_valid_before(self, time): + """ + Sets the certificate activation time. + """ + if not isinstance(time, datetime.datetime): + raise TypeError("Expecting datetime object.") + if self._not_valid_before is not None: + raise ValueError("The not valid before may only be set once.") + time = _convert_to_naive_utc_time(time) + if time < _EARLIEST_UTC_TIME: + raise ValueError( + "The not valid before date must be on or after" + " 1950 January 1)." + ) + if self._not_valid_after is not None and time > self._not_valid_after: + raise ValueError( + "The not valid before date must be before the not valid after " + "date." + ) + return CertificateBuilder( + self._issuer_name, + self._subject_name, + self._public_key, + self._serial_number, + time, + self._not_valid_after, + self._extensions, + ) + + def not_valid_after(self, time): + """ + Sets the certificate expiration time. + """ + if not isinstance(time, datetime.datetime): + raise TypeError("Expecting datetime object.") + if self._not_valid_after is not None: + raise ValueError("The not valid after may only be set once.") + time = _convert_to_naive_utc_time(time) + if time < _EARLIEST_UTC_TIME: + raise ValueError( + "The not valid after date must be on or after" + " 1950 January 1." + ) + if ( + self._not_valid_before is not None + and time < self._not_valid_before + ): + raise ValueError( + "The not valid after date must be after the not valid before " + "date." + ) + return CertificateBuilder( + self._issuer_name, + self._subject_name, + self._public_key, + self._serial_number, + self._not_valid_before, + time, + self._extensions, + ) + + def add_extension(self, extension, critical): + """ + Adds an X.509 extension to the certificate. + """ + if not isinstance(extension, ExtensionType): + raise TypeError("extension must be an ExtensionType") + + extension = Extension(extension.oid, critical, extension) + _reject_duplicate_extension(extension, self._extensions) + + return CertificateBuilder( + self._issuer_name, + self._subject_name, + self._public_key, + self._serial_number, + self._not_valid_before, + self._not_valid_after, + self._extensions + [extension], + ) + + def sign(self, private_key, algorithm, backend=None): + """ + Signs the certificate using the CA's private key. + """ + backend = _get_backend(backend) + if self._subject_name is None: + raise ValueError("A certificate must have a subject name") + + if self._issuer_name is None: + raise ValueError("A certificate must have an issuer name") + + if self._serial_number is None: + raise ValueError("A certificate must have a serial number") + + if self._not_valid_before is None: + raise ValueError("A certificate must have a not valid before time") + + if self._not_valid_after is None: + raise ValueError("A certificate must have a not valid after time") + + if self._public_key is None: + raise ValueError("A certificate must have a public key") + + return backend.create_x509_certificate(self, private_key, algorithm) + + +class CertificateRevocationListBuilder(object): + def __init__( + self, + issuer_name=None, + last_update=None, + next_update=None, + extensions=[], + revoked_certificates=[], + ): + self._issuer_name = issuer_name + self._last_update = last_update + self._next_update = next_update + self._extensions = extensions + self._revoked_certificates = revoked_certificates + + def issuer_name(self, issuer_name): + if not isinstance(issuer_name, Name): + raise TypeError("Expecting x509.Name object.") + if self._issuer_name is not None: + raise ValueError("The issuer name may only be set once.") + return CertificateRevocationListBuilder( + issuer_name, + self._last_update, + self._next_update, + self._extensions, + self._revoked_certificates, + ) + + def last_update(self, last_update): + if not isinstance(last_update, datetime.datetime): + raise TypeError("Expecting datetime object.") + if self._last_update is not None: + raise ValueError("Last update may only be set once.") + last_update = _convert_to_naive_utc_time(last_update) + if last_update < _EARLIEST_UTC_TIME: + raise ValueError( + "The last update date must be on or after" " 1950 January 1." + ) + if self._next_update is not None and last_update > self._next_update: + raise ValueError( + "The last update date must be before the next update date." + ) + return CertificateRevocationListBuilder( + self._issuer_name, + last_update, + self._next_update, + self._extensions, + self._revoked_certificates, + ) + + def next_update(self, next_update): + if not isinstance(next_update, datetime.datetime): + raise TypeError("Expecting datetime object.") + if self._next_update is not None: + raise ValueError("Last update may only be set once.") + next_update = _convert_to_naive_utc_time(next_update) + if next_update < _EARLIEST_UTC_TIME: + raise ValueError( + "The last update date must be on or after" " 1950 January 1." + ) + if self._last_update is not None and next_update < self._last_update: + raise ValueError( + "The next update date must be after the last update date." + ) + return CertificateRevocationListBuilder( + self._issuer_name, + self._last_update, + next_update, + self._extensions, + self._revoked_certificates, + ) + + def add_extension(self, extension, critical): + """ + Adds an X.509 extension to the certificate revocation list. + """ + if not isinstance(extension, ExtensionType): + raise TypeError("extension must be an ExtensionType") + + extension = Extension(extension.oid, critical, extension) + _reject_duplicate_extension(extension, self._extensions) + return CertificateRevocationListBuilder( + self._issuer_name, + self._last_update, + self._next_update, + self._extensions + [extension], + self._revoked_certificates, + ) + + def add_revoked_certificate(self, revoked_certificate): + """ + Adds a revoked certificate to the CRL. + """ + if not isinstance(revoked_certificate, RevokedCertificate): + raise TypeError("Must be an instance of RevokedCertificate") + + return CertificateRevocationListBuilder( + self._issuer_name, + self._last_update, + self._next_update, + self._extensions, + self._revoked_certificates + [revoked_certificate], + ) + + def sign(self, private_key, algorithm, backend=None): + backend = _get_backend(backend) + if self._issuer_name is None: + raise ValueError("A CRL must have an issuer name") + + if self._last_update is None: + raise ValueError("A CRL must have a last update time") + + if self._next_update is None: + raise ValueError("A CRL must have a next update time") + + return backend.create_x509_crl(self, private_key, algorithm) + + +class RevokedCertificateBuilder(object): + def __init__( + self, serial_number=None, revocation_date=None, extensions=[] + ): + self._serial_number = serial_number + self._revocation_date = revocation_date + self._extensions = extensions + + def serial_number(self, number): + if not isinstance(number, six.integer_types): + raise TypeError("Serial number must be of integral type.") + if self._serial_number is not None: + raise ValueError("The serial number may only be set once.") + if number <= 0: + raise ValueError("The serial number should be positive") + + # ASN.1 integers are always signed, so most significant bit must be + # zero. + if number.bit_length() >= 160: # As defined in RFC 5280 + raise ValueError( + "The serial number should not be more than 159 " "bits." + ) + return RevokedCertificateBuilder( + number, self._revocation_date, self._extensions + ) + + def revocation_date(self, time): + if not isinstance(time, datetime.datetime): + raise TypeError("Expecting datetime object.") + if self._revocation_date is not None: + raise ValueError("The revocation date may only be set once.") + time = _convert_to_naive_utc_time(time) + if time < _EARLIEST_UTC_TIME: + raise ValueError( + "The revocation date must be on or after" " 1950 January 1." + ) + return RevokedCertificateBuilder( + self._serial_number, time, self._extensions + ) + + def add_extension(self, extension, critical): + if not isinstance(extension, ExtensionType): + raise TypeError("extension must be an ExtensionType") + + extension = Extension(extension.oid, critical, extension) + _reject_duplicate_extension(extension, self._extensions) + return RevokedCertificateBuilder( + self._serial_number, + self._revocation_date, + self._extensions + [extension], + ) + + def build(self, backend=None): + backend = _get_backend(backend) + if self._serial_number is None: + raise ValueError("A revoked certificate must have a serial number") + if self._revocation_date is None: + raise ValueError( + "A revoked certificate must have a revocation date" + ) + + return backend.create_x509_revoked_certificate(self) + + +def random_serial_number(): + return utils.int_from_bytes(os.urandom(20), "big") >> 1 diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/x509/certificate_transparency.py b/WebCrawler/venv/Lib/site-packages/cryptography/x509/certificate_transparency.py new file mode 100644 index 0000000..d00fe81 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/cryptography/x509/certificate_transparency.py @@ -0,0 +1,46 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function + +import abc +from enum import Enum + +import six + + +class LogEntryType(Enum): + X509_CERTIFICATE = 0 + PRE_CERTIFICATE = 1 + + +class Version(Enum): + v1 = 0 + + +@six.add_metaclass(abc.ABCMeta) +class SignedCertificateTimestamp(object): + @abc.abstractproperty + def version(self): + """ + Returns the SCT version. + """ + + @abc.abstractproperty + def log_id(self): + """ + Returns an identifier indicating which log this SCT is for. + """ + + @abc.abstractproperty + def timestamp(self): + """ + Returns the timestamp for this SCT. + """ + + @abc.abstractproperty + def entry_type(self): + """ + Returns whether this is an SCT for a certificate or pre-certificate. + """ diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/x509/extensions.py b/WebCrawler/venv/Lib/site-packages/cryptography/x509/extensions.py new file mode 100644 index 0000000..130ba69 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/cryptography/x509/extensions.py @@ -0,0 +1,1702 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function + +import abc +import datetime +import hashlib +import ipaddress +from enum import Enum + +import six + +from cryptography import utils +from cryptography.hazmat._der import ( + BIT_STRING, + DERReader, + OBJECT_IDENTIFIER, + SEQUENCE, +) +from cryptography.hazmat.primitives import constant_time, serialization +from cryptography.hazmat.primitives.asymmetric.ec import EllipticCurvePublicKey +from cryptography.hazmat.primitives.asymmetric.rsa import RSAPublicKey +from cryptography.x509.certificate_transparency import ( + SignedCertificateTimestamp, +) +from cryptography.x509.general_name import GeneralName, IPAddress, OtherName +from cryptography.x509.name import RelativeDistinguishedName +from cryptography.x509.oid import ( + CRLEntryExtensionOID, + ExtensionOID, + OCSPExtensionOID, + ObjectIdentifier, +) + + +def _key_identifier_from_public_key(public_key): + if isinstance(public_key, RSAPublicKey): + data = public_key.public_bytes( + serialization.Encoding.DER, + serialization.PublicFormat.PKCS1, + ) + elif isinstance(public_key, EllipticCurvePublicKey): + data = public_key.public_bytes( + serialization.Encoding.X962, + serialization.PublicFormat.UncompressedPoint, + ) + else: + # This is a very slow way to do this. + serialized = public_key.public_bytes( + serialization.Encoding.DER, + serialization.PublicFormat.SubjectPublicKeyInfo, + ) + + reader = DERReader(serialized) + with reader.read_single_element(SEQUENCE) as public_key_info: + algorithm = public_key_info.read_element(SEQUENCE) + public_key = public_key_info.read_element(BIT_STRING) + + # Double-check the algorithm structure. + with algorithm: + algorithm.read_element(OBJECT_IDENTIFIER) + if not algorithm.is_empty(): + # Skip the optional parameters field. + algorithm.read_any_element() + + # BIT STRING contents begin with the number of padding bytes added. It + # must be zero for SubjectPublicKeyInfo structures. + if public_key.read_byte() != 0: + raise ValueError("Invalid public key encoding") + + data = public_key.data + + return hashlib.sha1(data).digest() + + +def _make_sequence_methods(field_name): + def len_method(self): + return len(getattr(self, field_name)) + + def iter_method(self): + return iter(getattr(self, field_name)) + + def getitem_method(self, idx): + return getattr(self, field_name)[idx] + + return len_method, iter_method, getitem_method + + +class DuplicateExtension(Exception): + def __init__(self, msg, oid): + super(DuplicateExtension, self).__init__(msg) + self.oid = oid + + +class ExtensionNotFound(Exception): + def __init__(self, msg, oid): + super(ExtensionNotFound, self).__init__(msg) + self.oid = oid + + +@six.add_metaclass(abc.ABCMeta) +class ExtensionType(object): + @abc.abstractproperty + def oid(self): + """ + Returns the oid associated with the given extension type. + """ + + +class Extensions(object): + def __init__(self, extensions): + self._extensions = extensions + + def get_extension_for_oid(self, oid): + for ext in self: + if ext.oid == oid: + return ext + + raise ExtensionNotFound("No {} extension was found".format(oid), oid) + + def get_extension_for_class(self, extclass): + if extclass is UnrecognizedExtension: + raise TypeError( + "UnrecognizedExtension can't be used with " + "get_extension_for_class because more than one instance of the" + " class may be present." + ) + + for ext in self: + if isinstance(ext.value, extclass): + return ext + + raise ExtensionNotFound( + "No {} extension was found".format(extclass), extclass.oid + ) + + __len__, __iter__, __getitem__ = _make_sequence_methods("_extensions") + + def __repr__(self): + return "".format(self._extensions) + + +@utils.register_interface(ExtensionType) +class CRLNumber(object): + oid = ExtensionOID.CRL_NUMBER + + def __init__(self, crl_number): + if not isinstance(crl_number, six.integer_types): + raise TypeError("crl_number must be an integer") + + self._crl_number = crl_number + + def __eq__(self, other): + if not isinstance(other, CRLNumber): + return NotImplemented + + return self.crl_number == other.crl_number + + def __ne__(self, other): + return not self == other + + def __hash__(self): + return hash(self.crl_number) + + def __repr__(self): + return "".format(self.crl_number) + + crl_number = utils.read_only_property("_crl_number") + + +@utils.register_interface(ExtensionType) +class AuthorityKeyIdentifier(object): + oid = ExtensionOID.AUTHORITY_KEY_IDENTIFIER + + def __init__( + self, + key_identifier, + authority_cert_issuer, + authority_cert_serial_number, + ): + if (authority_cert_issuer is None) != ( + authority_cert_serial_number is None + ): + raise ValueError( + "authority_cert_issuer and authority_cert_serial_number " + "must both be present or both None" + ) + + if authority_cert_issuer is not None: + authority_cert_issuer = list(authority_cert_issuer) + if not all( + isinstance(x, GeneralName) for x in authority_cert_issuer + ): + raise TypeError( + "authority_cert_issuer must be a list of GeneralName " + "objects" + ) + + if authority_cert_serial_number is not None and not isinstance( + authority_cert_serial_number, six.integer_types + ): + raise TypeError("authority_cert_serial_number must be an integer") + + self._key_identifier = key_identifier + self._authority_cert_issuer = authority_cert_issuer + self._authority_cert_serial_number = authority_cert_serial_number + + @classmethod + def from_issuer_public_key(cls, public_key): + digest = _key_identifier_from_public_key(public_key) + return cls( + key_identifier=digest, + authority_cert_issuer=None, + authority_cert_serial_number=None, + ) + + @classmethod + def from_issuer_subject_key_identifier(cls, ski): + return cls( + key_identifier=ski.digest, + authority_cert_issuer=None, + authority_cert_serial_number=None, + ) + + def __repr__(self): + return ( + "".format(self) + ) + + def __eq__(self, other): + if not isinstance(other, AuthorityKeyIdentifier): + return NotImplemented + + return ( + self.key_identifier == other.key_identifier + and self.authority_cert_issuer == other.authority_cert_issuer + and self.authority_cert_serial_number + == other.authority_cert_serial_number + ) + + def __ne__(self, other): + return not self == other + + def __hash__(self): + if self.authority_cert_issuer is None: + aci = None + else: + aci = tuple(self.authority_cert_issuer) + return hash( + (self.key_identifier, aci, self.authority_cert_serial_number) + ) + + key_identifier = utils.read_only_property("_key_identifier") + authority_cert_issuer = utils.read_only_property("_authority_cert_issuer") + authority_cert_serial_number = utils.read_only_property( + "_authority_cert_serial_number" + ) + + +@utils.register_interface(ExtensionType) +class SubjectKeyIdentifier(object): + oid = ExtensionOID.SUBJECT_KEY_IDENTIFIER + + def __init__(self, digest): + self._digest = digest + + @classmethod + def from_public_key(cls, public_key): + return cls(_key_identifier_from_public_key(public_key)) + + digest = utils.read_only_property("_digest") + + def __repr__(self): + return "".format(self.digest) + + def __eq__(self, other): + if not isinstance(other, SubjectKeyIdentifier): + return NotImplemented + + return constant_time.bytes_eq(self.digest, other.digest) + + def __ne__(self, other): + return not self == other + + def __hash__(self): + return hash(self.digest) + + +@utils.register_interface(ExtensionType) +class AuthorityInformationAccess(object): + oid = ExtensionOID.AUTHORITY_INFORMATION_ACCESS + + def __init__(self, descriptions): + descriptions = list(descriptions) + if not all(isinstance(x, AccessDescription) for x in descriptions): + raise TypeError( + "Every item in the descriptions list must be an " + "AccessDescription" + ) + + self._descriptions = descriptions + + __len__, __iter__, __getitem__ = _make_sequence_methods("_descriptions") + + def __repr__(self): + return "".format(self._descriptions) + + def __eq__(self, other): + if not isinstance(other, AuthorityInformationAccess): + return NotImplemented + + return self._descriptions == other._descriptions + + def __ne__(self, other): + return not self == other + + def __hash__(self): + return hash(tuple(self._descriptions)) + + +@utils.register_interface(ExtensionType) +class SubjectInformationAccess(object): + oid = ExtensionOID.SUBJECT_INFORMATION_ACCESS + + def __init__(self, descriptions): + descriptions = list(descriptions) + if not all(isinstance(x, AccessDescription) for x in descriptions): + raise TypeError( + "Every item in the descriptions list must be an " + "AccessDescription" + ) + + self._descriptions = descriptions + + __len__, __iter__, __getitem__ = _make_sequence_methods("_descriptions") + + def __repr__(self): + return "".format(self._descriptions) + + def __eq__(self, other): + if not isinstance(other, SubjectInformationAccess): + return NotImplemented + + return self._descriptions == other._descriptions + + def __ne__(self, other): + return not self == other + + def __hash__(self): + return hash(tuple(self._descriptions)) + + +class AccessDescription(object): + def __init__(self, access_method, access_location): + if not isinstance(access_method, ObjectIdentifier): + raise TypeError("access_method must be an ObjectIdentifier") + + if not isinstance(access_location, GeneralName): + raise TypeError("access_location must be a GeneralName") + + self._access_method = access_method + self._access_location = access_location + + def __repr__(self): + return ( + "".format(self) + ) + + def __eq__(self, other): + if not isinstance(other, AccessDescription): + return NotImplemented + + return ( + self.access_method == other.access_method + and self.access_location == other.access_location + ) + + def __ne__(self, other): + return not self == other + + def __hash__(self): + return hash((self.access_method, self.access_location)) + + access_method = utils.read_only_property("_access_method") + access_location = utils.read_only_property("_access_location") + + +@utils.register_interface(ExtensionType) +class BasicConstraints(object): + oid = ExtensionOID.BASIC_CONSTRAINTS + + def __init__(self, ca, path_length): + if not isinstance(ca, bool): + raise TypeError("ca must be a boolean value") + + if path_length is not None and not ca: + raise ValueError("path_length must be None when ca is False") + + if path_length is not None and ( + not isinstance(path_length, six.integer_types) or path_length < 0 + ): + raise TypeError( + "path_length must be a non-negative integer or None" + ) + + self._ca = ca + self._path_length = path_length + + ca = utils.read_only_property("_ca") + path_length = utils.read_only_property("_path_length") + + def __repr__(self): + return ( + "" + ).format(self) + + def __eq__(self, other): + if not isinstance(other, BasicConstraints): + return NotImplemented + + return self.ca == other.ca and self.path_length == other.path_length + + def __ne__(self, other): + return not self == other + + def __hash__(self): + return hash((self.ca, self.path_length)) + + +@utils.register_interface(ExtensionType) +class DeltaCRLIndicator(object): + oid = ExtensionOID.DELTA_CRL_INDICATOR + + def __init__(self, crl_number): + if not isinstance(crl_number, six.integer_types): + raise TypeError("crl_number must be an integer") + + self._crl_number = crl_number + + crl_number = utils.read_only_property("_crl_number") + + def __eq__(self, other): + if not isinstance(other, DeltaCRLIndicator): + return NotImplemented + + return self.crl_number == other.crl_number + + def __ne__(self, other): + return not self == other + + def __hash__(self): + return hash(self.crl_number) + + def __repr__(self): + return "".format(self) + + +@utils.register_interface(ExtensionType) +class CRLDistributionPoints(object): + oid = ExtensionOID.CRL_DISTRIBUTION_POINTS + + def __init__(self, distribution_points): + distribution_points = list(distribution_points) + if not all( + isinstance(x, DistributionPoint) for x in distribution_points + ): + raise TypeError( + "distribution_points must be a list of DistributionPoint " + "objects" + ) + + self._distribution_points = distribution_points + + __len__, __iter__, __getitem__ = _make_sequence_methods( + "_distribution_points" + ) + + def __repr__(self): + return "".format(self._distribution_points) + + def __eq__(self, other): + if not isinstance(other, CRLDistributionPoints): + return NotImplemented + + return self._distribution_points == other._distribution_points + + def __ne__(self, other): + return not self == other + + def __hash__(self): + return hash(tuple(self._distribution_points)) + + +@utils.register_interface(ExtensionType) +class FreshestCRL(object): + oid = ExtensionOID.FRESHEST_CRL + + def __init__(self, distribution_points): + distribution_points = list(distribution_points) + if not all( + isinstance(x, DistributionPoint) for x in distribution_points + ): + raise TypeError( + "distribution_points must be a list of DistributionPoint " + "objects" + ) + + self._distribution_points = distribution_points + + __len__, __iter__, __getitem__ = _make_sequence_methods( + "_distribution_points" + ) + + def __repr__(self): + return "".format(self._distribution_points) + + def __eq__(self, other): + if not isinstance(other, FreshestCRL): + return NotImplemented + + return self._distribution_points == other._distribution_points + + def __ne__(self, other): + return not self == other + + def __hash__(self): + return hash(tuple(self._distribution_points)) + + +class DistributionPoint(object): + def __init__(self, full_name, relative_name, reasons, crl_issuer): + if full_name and relative_name: + raise ValueError( + "You cannot provide both full_name and relative_name, at " + "least one must be None." + ) + + if full_name: + full_name = list(full_name) + if not all(isinstance(x, GeneralName) for x in full_name): + raise TypeError( + "full_name must be a list of GeneralName objects" + ) + + if relative_name: + if not isinstance(relative_name, RelativeDistinguishedName): + raise TypeError( + "relative_name must be a RelativeDistinguishedName" + ) + + if crl_issuer: + crl_issuer = list(crl_issuer) + if not all(isinstance(x, GeneralName) for x in crl_issuer): + raise TypeError( + "crl_issuer must be None or a list of general names" + ) + + if reasons and ( + not isinstance(reasons, frozenset) + or not all(isinstance(x, ReasonFlags) for x in reasons) + ): + raise TypeError("reasons must be None or frozenset of ReasonFlags") + + if reasons and ( + ReasonFlags.unspecified in reasons + or ReasonFlags.remove_from_crl in reasons + ): + raise ValueError( + "unspecified and remove_from_crl are not valid reasons in a " + "DistributionPoint" + ) + + if reasons and not crl_issuer and not (full_name or relative_name): + raise ValueError( + "You must supply crl_issuer, full_name, or relative_name when " + "reasons is not None" + ) + + self._full_name = full_name + self._relative_name = relative_name + self._reasons = reasons + self._crl_issuer = crl_issuer + + def __repr__(self): + return ( + "".format(self) + ) + + def __eq__(self, other): + if not isinstance(other, DistributionPoint): + return NotImplemented + + return ( + self.full_name == other.full_name + and self.relative_name == other.relative_name + and self.reasons == other.reasons + and self.crl_issuer == other.crl_issuer + ) + + def __ne__(self, other): + return not self == other + + def __hash__(self): + if self.full_name is not None: + fn = tuple(self.full_name) + else: + fn = None + + if self.crl_issuer is not None: + crl_issuer = tuple(self.crl_issuer) + else: + crl_issuer = None + + return hash((fn, self.relative_name, self.reasons, crl_issuer)) + + full_name = utils.read_only_property("_full_name") + relative_name = utils.read_only_property("_relative_name") + reasons = utils.read_only_property("_reasons") + crl_issuer = utils.read_only_property("_crl_issuer") + + +class ReasonFlags(Enum): + unspecified = "unspecified" + key_compromise = "keyCompromise" + ca_compromise = "cACompromise" + affiliation_changed = "affiliationChanged" + superseded = "superseded" + cessation_of_operation = "cessationOfOperation" + certificate_hold = "certificateHold" + privilege_withdrawn = "privilegeWithdrawn" + aa_compromise = "aACompromise" + remove_from_crl = "removeFromCRL" + + +@utils.register_interface(ExtensionType) +class PolicyConstraints(object): + oid = ExtensionOID.POLICY_CONSTRAINTS + + def __init__(self, require_explicit_policy, inhibit_policy_mapping): + if require_explicit_policy is not None and not isinstance( + require_explicit_policy, six.integer_types + ): + raise TypeError( + "require_explicit_policy must be a non-negative integer or " + "None" + ) + + if inhibit_policy_mapping is not None and not isinstance( + inhibit_policy_mapping, six.integer_types + ): + raise TypeError( + "inhibit_policy_mapping must be a non-negative integer or None" + ) + + if inhibit_policy_mapping is None and require_explicit_policy is None: + raise ValueError( + "At least one of require_explicit_policy and " + "inhibit_policy_mapping must not be None" + ) + + self._require_explicit_policy = require_explicit_policy + self._inhibit_policy_mapping = inhibit_policy_mapping + + def __repr__(self): + return ( + u"".format(self) + ) + + def __eq__(self, other): + if not isinstance(other, PolicyConstraints): + return NotImplemented + + return ( + self.require_explicit_policy == other.require_explicit_policy + and self.inhibit_policy_mapping == other.inhibit_policy_mapping + ) + + def __ne__(self, other): + return not self == other + + def __hash__(self): + return hash( + (self.require_explicit_policy, self.inhibit_policy_mapping) + ) + + require_explicit_policy = utils.read_only_property( + "_require_explicit_policy" + ) + inhibit_policy_mapping = utils.read_only_property( + "_inhibit_policy_mapping" + ) + + +@utils.register_interface(ExtensionType) +class CertificatePolicies(object): + oid = ExtensionOID.CERTIFICATE_POLICIES + + def __init__(self, policies): + policies = list(policies) + if not all(isinstance(x, PolicyInformation) for x in policies): + raise TypeError( + "Every item in the policies list must be a " + "PolicyInformation" + ) + + self._policies = policies + + __len__, __iter__, __getitem__ = _make_sequence_methods("_policies") + + def __repr__(self): + return "".format(self._policies) + + def __eq__(self, other): + if not isinstance(other, CertificatePolicies): + return NotImplemented + + return self._policies == other._policies + + def __ne__(self, other): + return not self == other + + def __hash__(self): + return hash(tuple(self._policies)) + + +class PolicyInformation(object): + def __init__(self, policy_identifier, policy_qualifiers): + if not isinstance(policy_identifier, ObjectIdentifier): + raise TypeError("policy_identifier must be an ObjectIdentifier") + + self._policy_identifier = policy_identifier + + if policy_qualifiers: + policy_qualifiers = list(policy_qualifiers) + if not all( + isinstance(x, (six.text_type, UserNotice)) + for x in policy_qualifiers + ): + raise TypeError( + "policy_qualifiers must be a list of strings and/or " + "UserNotice objects or None" + ) + + self._policy_qualifiers = policy_qualifiers + + def __repr__(self): + return ( + "".format(self) + ) + + def __eq__(self, other): + if not isinstance(other, PolicyInformation): + return NotImplemented + + return ( + self.policy_identifier == other.policy_identifier + and self.policy_qualifiers == other.policy_qualifiers + ) + + def __ne__(self, other): + return not self == other + + def __hash__(self): + if self.policy_qualifiers is not None: + pq = tuple(self.policy_qualifiers) + else: + pq = None + + return hash((self.policy_identifier, pq)) + + policy_identifier = utils.read_only_property("_policy_identifier") + policy_qualifiers = utils.read_only_property("_policy_qualifiers") + + +class UserNotice(object): + def __init__(self, notice_reference, explicit_text): + if notice_reference and not isinstance( + notice_reference, NoticeReference + ): + raise TypeError( + "notice_reference must be None or a NoticeReference" + ) + + self._notice_reference = notice_reference + self._explicit_text = explicit_text + + def __repr__(self): + return ( + "".format(self) + ) + + def __eq__(self, other): + if not isinstance(other, UserNotice): + return NotImplemented + + return ( + self.notice_reference == other.notice_reference + and self.explicit_text == other.explicit_text + ) + + def __ne__(self, other): + return not self == other + + def __hash__(self): + return hash((self.notice_reference, self.explicit_text)) + + notice_reference = utils.read_only_property("_notice_reference") + explicit_text = utils.read_only_property("_explicit_text") + + +class NoticeReference(object): + def __init__(self, organization, notice_numbers): + self._organization = organization + notice_numbers = list(notice_numbers) + if not all(isinstance(x, int) for x in notice_numbers): + raise TypeError("notice_numbers must be a list of integers") + + self._notice_numbers = notice_numbers + + def __repr__(self): + return ( + "".format(self) + ) + + def __eq__(self, other): + if not isinstance(other, NoticeReference): + return NotImplemented + + return ( + self.organization == other.organization + and self.notice_numbers == other.notice_numbers + ) + + def __ne__(self, other): + return not self == other + + def __hash__(self): + return hash((self.organization, tuple(self.notice_numbers))) + + organization = utils.read_only_property("_organization") + notice_numbers = utils.read_only_property("_notice_numbers") + + +@utils.register_interface(ExtensionType) +class ExtendedKeyUsage(object): + oid = ExtensionOID.EXTENDED_KEY_USAGE + + def __init__(self, usages): + usages = list(usages) + if not all(isinstance(x, ObjectIdentifier) for x in usages): + raise TypeError( + "Every item in the usages list must be an ObjectIdentifier" + ) + + self._usages = usages + + __len__, __iter__, __getitem__ = _make_sequence_methods("_usages") + + def __repr__(self): + return "".format(self._usages) + + def __eq__(self, other): + if not isinstance(other, ExtendedKeyUsage): + return NotImplemented + + return self._usages == other._usages + + def __ne__(self, other): + return not self == other + + def __hash__(self): + return hash(tuple(self._usages)) + + +@utils.register_interface(ExtensionType) +class OCSPNoCheck(object): + oid = ExtensionOID.OCSP_NO_CHECK + + def __eq__(self, other): + if not isinstance(other, OCSPNoCheck): + return NotImplemented + + return True + + def __ne__(self, other): + return not self == other + + def __hash__(self): + return hash(OCSPNoCheck) + + def __repr__(self): + return "" + + +@utils.register_interface(ExtensionType) +class PrecertPoison(object): + oid = ExtensionOID.PRECERT_POISON + + def __eq__(self, other): + if not isinstance(other, PrecertPoison): + return NotImplemented + + return True + + def __ne__(self, other): + return not self == other + + def __hash__(self): + return hash(PrecertPoison) + + def __repr__(self): + return "" + + +@utils.register_interface(ExtensionType) +class TLSFeature(object): + oid = ExtensionOID.TLS_FEATURE + + def __init__(self, features): + features = list(features) + if ( + not all(isinstance(x, TLSFeatureType) for x in features) + or len(features) == 0 + ): + raise TypeError( + "features must be a list of elements from the TLSFeatureType " + "enum" + ) + + self._features = features + + __len__, __iter__, __getitem__ = _make_sequence_methods("_features") + + def __repr__(self): + return "".format(self) + + def __eq__(self, other): + if not isinstance(other, TLSFeature): + return NotImplemented + + return self._features == other._features + + def __ne__(self, other): + return not self == other + + def __hash__(self): + return hash(tuple(self._features)) + + +class TLSFeatureType(Enum): + # status_request is defined in RFC 6066 and is used for what is commonly + # called OCSP Must-Staple when present in the TLS Feature extension in an + # X.509 certificate. + status_request = 5 + # status_request_v2 is defined in RFC 6961 and allows multiple OCSP + # responses to be provided. It is not currently in use by clients or + # servers. + status_request_v2 = 17 + + +_TLS_FEATURE_TYPE_TO_ENUM = {x.value: x for x in TLSFeatureType} + + +@utils.register_interface(ExtensionType) +class InhibitAnyPolicy(object): + oid = ExtensionOID.INHIBIT_ANY_POLICY + + def __init__(self, skip_certs): + if not isinstance(skip_certs, six.integer_types): + raise TypeError("skip_certs must be an integer") + + if skip_certs < 0: + raise ValueError("skip_certs must be a non-negative integer") + + self._skip_certs = skip_certs + + def __repr__(self): + return "".format(self) + + def __eq__(self, other): + if not isinstance(other, InhibitAnyPolicy): + return NotImplemented + + return self.skip_certs == other.skip_certs + + def __ne__(self, other): + return not self == other + + def __hash__(self): + return hash(self.skip_certs) + + skip_certs = utils.read_only_property("_skip_certs") + + +@utils.register_interface(ExtensionType) +class KeyUsage(object): + oid = ExtensionOID.KEY_USAGE + + def __init__( + self, + digital_signature, + content_commitment, + key_encipherment, + data_encipherment, + key_agreement, + key_cert_sign, + crl_sign, + encipher_only, + decipher_only, + ): + if not key_agreement and (encipher_only or decipher_only): + raise ValueError( + "encipher_only and decipher_only can only be true when " + "key_agreement is true" + ) + + self._digital_signature = digital_signature + self._content_commitment = content_commitment + self._key_encipherment = key_encipherment + self._data_encipherment = data_encipherment + self._key_agreement = key_agreement + self._key_cert_sign = key_cert_sign + self._crl_sign = crl_sign + self._encipher_only = encipher_only + self._decipher_only = decipher_only + + digital_signature = utils.read_only_property("_digital_signature") + content_commitment = utils.read_only_property("_content_commitment") + key_encipherment = utils.read_only_property("_key_encipherment") + data_encipherment = utils.read_only_property("_data_encipherment") + key_agreement = utils.read_only_property("_key_agreement") + key_cert_sign = utils.read_only_property("_key_cert_sign") + crl_sign = utils.read_only_property("_crl_sign") + + @property + def encipher_only(self): + if not self.key_agreement: + raise ValueError( + "encipher_only is undefined unless key_agreement is true" + ) + else: + return self._encipher_only + + @property + def decipher_only(self): + if not self.key_agreement: + raise ValueError( + "decipher_only is undefined unless key_agreement is true" + ) + else: + return self._decipher_only + + def __repr__(self): + try: + encipher_only = self.encipher_only + decipher_only = self.decipher_only + except ValueError: + # Users found None confusing because even though encipher/decipher + # have no meaning unless key_agreement is true, to construct an + # instance of the class you still need to pass False. + encipher_only = False + decipher_only = False + + return ( + "" + ).format(self, encipher_only, decipher_only) + + def __eq__(self, other): + if not isinstance(other, KeyUsage): + return NotImplemented + + return ( + self.digital_signature == other.digital_signature + and self.content_commitment == other.content_commitment + and self.key_encipherment == other.key_encipherment + and self.data_encipherment == other.data_encipherment + and self.key_agreement == other.key_agreement + and self.key_cert_sign == other.key_cert_sign + and self.crl_sign == other.crl_sign + and self._encipher_only == other._encipher_only + and self._decipher_only == other._decipher_only + ) + + def __ne__(self, other): + return not self == other + + def __hash__(self): + return hash( + ( + self.digital_signature, + self.content_commitment, + self.key_encipherment, + self.data_encipherment, + self.key_agreement, + self.key_cert_sign, + self.crl_sign, + self._encipher_only, + self._decipher_only, + ) + ) + + +@utils.register_interface(ExtensionType) +class NameConstraints(object): + oid = ExtensionOID.NAME_CONSTRAINTS + + def __init__(self, permitted_subtrees, excluded_subtrees): + if permitted_subtrees is not None: + permitted_subtrees = list(permitted_subtrees) + if not all(isinstance(x, GeneralName) for x in permitted_subtrees): + raise TypeError( + "permitted_subtrees must be a list of GeneralName objects " + "or None" + ) + + self._validate_ip_name(permitted_subtrees) + + if excluded_subtrees is not None: + excluded_subtrees = list(excluded_subtrees) + if not all(isinstance(x, GeneralName) for x in excluded_subtrees): + raise TypeError( + "excluded_subtrees must be a list of GeneralName objects " + "or None" + ) + + self._validate_ip_name(excluded_subtrees) + + if permitted_subtrees is None and excluded_subtrees is None: + raise ValueError( + "At least one of permitted_subtrees and excluded_subtrees " + "must not be None" + ) + + self._permitted_subtrees = permitted_subtrees + self._excluded_subtrees = excluded_subtrees + + def __eq__(self, other): + if not isinstance(other, NameConstraints): + return NotImplemented + + return ( + self.excluded_subtrees == other.excluded_subtrees + and self.permitted_subtrees == other.permitted_subtrees + ) + + def __ne__(self, other): + return not self == other + + def _validate_ip_name(self, tree): + if any( + isinstance(name, IPAddress) + and not isinstance( + name.value, (ipaddress.IPv4Network, ipaddress.IPv6Network) + ) + for name in tree + ): + raise TypeError( + "IPAddress name constraints must be an IPv4Network or" + " IPv6Network object" + ) + + def __repr__(self): + return ( + u"".format(self) + ) + + def __hash__(self): + if self.permitted_subtrees is not None: + ps = tuple(self.permitted_subtrees) + else: + ps = None + + if self.excluded_subtrees is not None: + es = tuple(self.excluded_subtrees) + else: + es = None + + return hash((ps, es)) + + permitted_subtrees = utils.read_only_property("_permitted_subtrees") + excluded_subtrees = utils.read_only_property("_excluded_subtrees") + + +class Extension(object): + def __init__(self, oid, critical, value): + if not isinstance(oid, ObjectIdentifier): + raise TypeError( + "oid argument must be an ObjectIdentifier instance." + ) + + if not isinstance(critical, bool): + raise TypeError("critical must be a boolean value") + + self._oid = oid + self._critical = critical + self._value = value + + oid = utils.read_only_property("_oid") + critical = utils.read_only_property("_critical") + value = utils.read_only_property("_value") + + def __repr__(self): + return ( + "" + ).format(self) + + def __eq__(self, other): + if not isinstance(other, Extension): + return NotImplemented + + return ( + self.oid == other.oid + and self.critical == other.critical + and self.value == other.value + ) + + def __ne__(self, other): + return not self == other + + def __hash__(self): + return hash((self.oid, self.critical, self.value)) + + +class GeneralNames(object): + def __init__(self, general_names): + general_names = list(general_names) + if not all(isinstance(x, GeneralName) for x in general_names): + raise TypeError( + "Every item in the general_names list must be an " + "object conforming to the GeneralName interface" + ) + + self._general_names = general_names + + __len__, __iter__, __getitem__ = _make_sequence_methods("_general_names") + + def get_values_for_type(self, type): + # Return the value of each GeneralName, except for OtherName instances + # which we return directly because it has two important properties not + # just one value. + objs = (i for i in self if isinstance(i, type)) + if type != OtherName: + objs = (i.value for i in objs) + return list(objs) + + def __repr__(self): + return "".format(self._general_names) + + def __eq__(self, other): + if not isinstance(other, GeneralNames): + return NotImplemented + + return self._general_names == other._general_names + + def __ne__(self, other): + return not self == other + + def __hash__(self): + return hash(tuple(self._general_names)) + + +@utils.register_interface(ExtensionType) +class SubjectAlternativeName(object): + oid = ExtensionOID.SUBJECT_ALTERNATIVE_NAME + + def __init__(self, general_names): + self._general_names = GeneralNames(general_names) + + __len__, __iter__, __getitem__ = _make_sequence_methods("_general_names") + + def get_values_for_type(self, type): + return self._general_names.get_values_for_type(type) + + def __repr__(self): + return "".format(self._general_names) + + def __eq__(self, other): + if not isinstance(other, SubjectAlternativeName): + return NotImplemented + + return self._general_names == other._general_names + + def __ne__(self, other): + return not self == other + + def __hash__(self): + return hash(self._general_names) + + +@utils.register_interface(ExtensionType) +class IssuerAlternativeName(object): + oid = ExtensionOID.ISSUER_ALTERNATIVE_NAME + + def __init__(self, general_names): + self._general_names = GeneralNames(general_names) + + __len__, __iter__, __getitem__ = _make_sequence_methods("_general_names") + + def get_values_for_type(self, type): + return self._general_names.get_values_for_type(type) + + def __repr__(self): + return "".format(self._general_names) + + def __eq__(self, other): + if not isinstance(other, IssuerAlternativeName): + return NotImplemented + + return self._general_names == other._general_names + + def __ne__(self, other): + return not self == other + + def __hash__(self): + return hash(self._general_names) + + +@utils.register_interface(ExtensionType) +class CertificateIssuer(object): + oid = CRLEntryExtensionOID.CERTIFICATE_ISSUER + + def __init__(self, general_names): + self._general_names = GeneralNames(general_names) + + __len__, __iter__, __getitem__ = _make_sequence_methods("_general_names") + + def get_values_for_type(self, type): + return self._general_names.get_values_for_type(type) + + def __repr__(self): + return "".format(self._general_names) + + def __eq__(self, other): + if not isinstance(other, CertificateIssuer): + return NotImplemented + + return self._general_names == other._general_names + + def __ne__(self, other): + return not self == other + + def __hash__(self): + return hash(self._general_names) + + +@utils.register_interface(ExtensionType) +class CRLReason(object): + oid = CRLEntryExtensionOID.CRL_REASON + + def __init__(self, reason): + if not isinstance(reason, ReasonFlags): + raise TypeError("reason must be an element from ReasonFlags") + + self._reason = reason + + def __repr__(self): + return "".format(self._reason) + + def __eq__(self, other): + if not isinstance(other, CRLReason): + return NotImplemented + + return self.reason == other.reason + + def __ne__(self, other): + return not self == other + + def __hash__(self): + return hash(self.reason) + + reason = utils.read_only_property("_reason") + + +@utils.register_interface(ExtensionType) +class InvalidityDate(object): + oid = CRLEntryExtensionOID.INVALIDITY_DATE + + def __init__(self, invalidity_date): + if not isinstance(invalidity_date, datetime.datetime): + raise TypeError("invalidity_date must be a datetime.datetime") + + self._invalidity_date = invalidity_date + + def __repr__(self): + return "".format( + self._invalidity_date + ) + + def __eq__(self, other): + if not isinstance(other, InvalidityDate): + return NotImplemented + + return self.invalidity_date == other.invalidity_date + + def __ne__(self, other): + return not self == other + + def __hash__(self): + return hash(self.invalidity_date) + + invalidity_date = utils.read_only_property("_invalidity_date") + + +@utils.register_interface(ExtensionType) +class PrecertificateSignedCertificateTimestamps(object): + oid = ExtensionOID.PRECERT_SIGNED_CERTIFICATE_TIMESTAMPS + + def __init__(self, signed_certificate_timestamps): + signed_certificate_timestamps = list(signed_certificate_timestamps) + if not all( + isinstance(sct, SignedCertificateTimestamp) + for sct in signed_certificate_timestamps + ): + raise TypeError( + "Every item in the signed_certificate_timestamps list must be " + "a SignedCertificateTimestamp" + ) + self._signed_certificate_timestamps = signed_certificate_timestamps + + __len__, __iter__, __getitem__ = _make_sequence_methods( + "_signed_certificate_timestamps" + ) + + def __repr__(self): + return "".format( + list(self) + ) + + def __hash__(self): + return hash(tuple(self._signed_certificate_timestamps)) + + def __eq__(self, other): + if not isinstance(other, PrecertificateSignedCertificateTimestamps): + return NotImplemented + + return ( + self._signed_certificate_timestamps + == other._signed_certificate_timestamps + ) + + def __ne__(self, other): + return not self == other + + +@utils.register_interface(ExtensionType) +class SignedCertificateTimestamps(object): + oid = ExtensionOID.SIGNED_CERTIFICATE_TIMESTAMPS + + def __init__(self, signed_certificate_timestamps): + signed_certificate_timestamps = list(signed_certificate_timestamps) + if not all( + isinstance(sct, SignedCertificateTimestamp) + for sct in signed_certificate_timestamps + ): + raise TypeError( + "Every item in the signed_certificate_timestamps list must be " + "a SignedCertificateTimestamp" + ) + self._signed_certificate_timestamps = signed_certificate_timestamps + + __len__, __iter__, __getitem__ = _make_sequence_methods( + "_signed_certificate_timestamps" + ) + + def __repr__(self): + return "".format(list(self)) + + def __hash__(self): + return hash(tuple(self._signed_certificate_timestamps)) + + def __eq__(self, other): + if not isinstance(other, SignedCertificateTimestamps): + return NotImplemented + + return ( + self._signed_certificate_timestamps + == other._signed_certificate_timestamps + ) + + def __ne__(self, other): + return not self == other + + +@utils.register_interface(ExtensionType) +class OCSPNonce(object): + oid = OCSPExtensionOID.NONCE + + def __init__(self, nonce): + if not isinstance(nonce, bytes): + raise TypeError("nonce must be bytes") + + self._nonce = nonce + + def __eq__(self, other): + if not isinstance(other, OCSPNonce): + return NotImplemented + + return self.nonce == other.nonce + + def __ne__(self, other): + return not self == other + + def __hash__(self): + return hash(self.nonce) + + def __repr__(self): + return "".format(self) + + nonce = utils.read_only_property("_nonce") + + +@utils.register_interface(ExtensionType) +class IssuingDistributionPoint(object): + oid = ExtensionOID.ISSUING_DISTRIBUTION_POINT + + def __init__( + self, + full_name, + relative_name, + only_contains_user_certs, + only_contains_ca_certs, + only_some_reasons, + indirect_crl, + only_contains_attribute_certs, + ): + if only_some_reasons and ( + not isinstance(only_some_reasons, frozenset) + or not all(isinstance(x, ReasonFlags) for x in only_some_reasons) + ): + raise TypeError( + "only_some_reasons must be None or frozenset of ReasonFlags" + ) + + if only_some_reasons and ( + ReasonFlags.unspecified in only_some_reasons + or ReasonFlags.remove_from_crl in only_some_reasons + ): + raise ValueError( + "unspecified and remove_from_crl are not valid reasons in an " + "IssuingDistributionPoint" + ) + + if not ( + isinstance(only_contains_user_certs, bool) + and isinstance(only_contains_ca_certs, bool) + and isinstance(indirect_crl, bool) + and isinstance(only_contains_attribute_certs, bool) + ): + raise TypeError( + "only_contains_user_certs, only_contains_ca_certs, " + "indirect_crl and only_contains_attribute_certs " + "must all be boolean." + ) + + crl_constraints = [ + only_contains_user_certs, + only_contains_ca_certs, + indirect_crl, + only_contains_attribute_certs, + ] + + if len([x for x in crl_constraints if x]) > 1: + raise ValueError( + "Only one of the following can be set to True: " + "only_contains_user_certs, only_contains_ca_certs, " + "indirect_crl, only_contains_attribute_certs" + ) + + if not any( + [ + only_contains_user_certs, + only_contains_ca_certs, + indirect_crl, + only_contains_attribute_certs, + full_name, + relative_name, + only_some_reasons, + ] + ): + raise ValueError( + "Cannot create empty extension: " + "if only_contains_user_certs, only_contains_ca_certs, " + "indirect_crl, and only_contains_attribute_certs are all False" + ", then either full_name, relative_name, or only_some_reasons " + "must have a value." + ) + + self._only_contains_user_certs = only_contains_user_certs + self._only_contains_ca_certs = only_contains_ca_certs + self._indirect_crl = indirect_crl + self._only_contains_attribute_certs = only_contains_attribute_certs + self._only_some_reasons = only_some_reasons + self._full_name = full_name + self._relative_name = relative_name + + def __repr__(self): + return ( + "".format(self) + ) + + def __eq__(self, other): + if not isinstance(other, IssuingDistributionPoint): + return NotImplemented + + return ( + self.full_name == other.full_name + and self.relative_name == other.relative_name + and self.only_contains_user_certs == other.only_contains_user_certs + and self.only_contains_ca_certs == other.only_contains_ca_certs + and self.only_some_reasons == other.only_some_reasons + and self.indirect_crl == other.indirect_crl + and self.only_contains_attribute_certs + == other.only_contains_attribute_certs + ) + + def __ne__(self, other): + return not self == other + + def __hash__(self): + return hash( + ( + self.full_name, + self.relative_name, + self.only_contains_user_certs, + self.only_contains_ca_certs, + self.only_some_reasons, + self.indirect_crl, + self.only_contains_attribute_certs, + ) + ) + + full_name = utils.read_only_property("_full_name") + relative_name = utils.read_only_property("_relative_name") + only_contains_user_certs = utils.read_only_property( + "_only_contains_user_certs" + ) + only_contains_ca_certs = utils.read_only_property( + "_only_contains_ca_certs" + ) + only_some_reasons = utils.read_only_property("_only_some_reasons") + indirect_crl = utils.read_only_property("_indirect_crl") + only_contains_attribute_certs = utils.read_only_property( + "_only_contains_attribute_certs" + ) + + +@utils.register_interface(ExtensionType) +class UnrecognizedExtension(object): + def __init__(self, oid, value): + if not isinstance(oid, ObjectIdentifier): + raise TypeError("oid must be an ObjectIdentifier") + self._oid = oid + self._value = value + + oid = utils.read_only_property("_oid") + value = utils.read_only_property("_value") + + def __repr__(self): + return ( + "".format(self) + ) + + def __eq__(self, other): + if not isinstance(other, UnrecognizedExtension): + return NotImplemented + + return self.oid == other.oid and self.value == other.value + + def __ne__(self, other): + return not self == other + + def __hash__(self): + return hash((self.oid, self.value)) diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/x509/general_name.py b/WebCrawler/venv/Lib/site-packages/cryptography/x509/general_name.py new file mode 100644 index 0000000..9be9d8c --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/cryptography/x509/general_name.py @@ -0,0 +1,294 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function + +import abc +import ipaddress +from email.utils import parseaddr + +import six + +from cryptography import utils +from cryptography.x509.name import Name +from cryptography.x509.oid import ObjectIdentifier + + +_GENERAL_NAMES = { + 0: "otherName", + 1: "rfc822Name", + 2: "dNSName", + 3: "x400Address", + 4: "directoryName", + 5: "ediPartyName", + 6: "uniformResourceIdentifier", + 7: "iPAddress", + 8: "registeredID", +} + + +class UnsupportedGeneralNameType(Exception): + def __init__(self, msg, type): + super(UnsupportedGeneralNameType, self).__init__(msg) + self.type = type + + +@six.add_metaclass(abc.ABCMeta) +class GeneralName(object): + @abc.abstractproperty + def value(self): + """ + Return the value of the object + """ + + +@utils.register_interface(GeneralName) +class RFC822Name(object): + def __init__(self, value): + if isinstance(value, six.text_type): + try: + value.encode("ascii") + except UnicodeEncodeError: + raise ValueError( + "RFC822Name values should be passed as an A-label string. " + "This means unicode characters should be encoded via " + "a library like idna." + ) + else: + raise TypeError("value must be string") + + name, address = parseaddr(value) + if name or not address: + # parseaddr has found a name (e.g. Name ) or the entire + # value is an empty string. + raise ValueError("Invalid rfc822name value") + + self._value = value + + value = utils.read_only_property("_value") + + @classmethod + def _init_without_validation(cls, value): + instance = cls.__new__(cls) + instance._value = value + return instance + + def __repr__(self): + return "".format(self.value) + + def __eq__(self, other): + if not isinstance(other, RFC822Name): + return NotImplemented + + return self.value == other.value + + def __ne__(self, other): + return not self == other + + def __hash__(self): + return hash(self.value) + + +@utils.register_interface(GeneralName) +class DNSName(object): + def __init__(self, value): + if isinstance(value, six.text_type): + try: + value.encode("ascii") + except UnicodeEncodeError: + raise ValueError( + "DNSName values should be passed as an A-label string. " + "This means unicode characters should be encoded via " + "a library like idna." + ) + else: + raise TypeError("value must be string") + + self._value = value + + value = utils.read_only_property("_value") + + @classmethod + def _init_without_validation(cls, value): + instance = cls.__new__(cls) + instance._value = value + return instance + + def __repr__(self): + return "".format(self.value) + + def __eq__(self, other): + if not isinstance(other, DNSName): + return NotImplemented + + return self.value == other.value + + def __ne__(self, other): + return not self == other + + def __hash__(self): + return hash(self.value) + + +@utils.register_interface(GeneralName) +class UniformResourceIdentifier(object): + def __init__(self, value): + if isinstance(value, six.text_type): + try: + value.encode("ascii") + except UnicodeEncodeError: + raise ValueError( + "URI values should be passed as an A-label string. " + "This means unicode characters should be encoded via " + "a library like idna." + ) + else: + raise TypeError("value must be string") + + self._value = value + + value = utils.read_only_property("_value") + + @classmethod + def _init_without_validation(cls, value): + instance = cls.__new__(cls) + instance._value = value + return instance + + def __repr__(self): + return "".format(self.value) + + def __eq__(self, other): + if not isinstance(other, UniformResourceIdentifier): + return NotImplemented + + return self.value == other.value + + def __ne__(self, other): + return not self == other + + def __hash__(self): + return hash(self.value) + + +@utils.register_interface(GeneralName) +class DirectoryName(object): + def __init__(self, value): + if not isinstance(value, Name): + raise TypeError("value must be a Name") + + self._value = value + + value = utils.read_only_property("_value") + + def __repr__(self): + return "".format(self.value) + + def __eq__(self, other): + if not isinstance(other, DirectoryName): + return NotImplemented + + return self.value == other.value + + def __ne__(self, other): + return not self == other + + def __hash__(self): + return hash(self.value) + + +@utils.register_interface(GeneralName) +class RegisteredID(object): + def __init__(self, value): + if not isinstance(value, ObjectIdentifier): + raise TypeError("value must be an ObjectIdentifier") + + self._value = value + + value = utils.read_only_property("_value") + + def __repr__(self): + return "".format(self.value) + + def __eq__(self, other): + if not isinstance(other, RegisteredID): + return NotImplemented + + return self.value == other.value + + def __ne__(self, other): + return not self == other + + def __hash__(self): + return hash(self.value) + + +@utils.register_interface(GeneralName) +class IPAddress(object): + def __init__(self, value): + if not isinstance( + value, + ( + ipaddress.IPv4Address, + ipaddress.IPv6Address, + ipaddress.IPv4Network, + ipaddress.IPv6Network, + ), + ): + raise TypeError( + "value must be an instance of ipaddress.IPv4Address, " + "ipaddress.IPv6Address, ipaddress.IPv4Network, or " + "ipaddress.IPv6Network" + ) + + self._value = value + + value = utils.read_only_property("_value") + + def __repr__(self): + return "".format(self.value) + + def __eq__(self, other): + if not isinstance(other, IPAddress): + return NotImplemented + + return self.value == other.value + + def __ne__(self, other): + return not self == other + + def __hash__(self): + return hash(self.value) + + +@utils.register_interface(GeneralName) +class OtherName(object): + def __init__(self, type_id, value): + if not isinstance(type_id, ObjectIdentifier): + raise TypeError("type_id must be an ObjectIdentifier") + if not isinstance(value, bytes): + raise TypeError("value must be a binary string") + + self._type_id = type_id + self._value = value + + type_id = utils.read_only_property("_type_id") + value = utils.read_only_property("_value") + + def __repr__(self): + return "".format( + self.type_id, self.value + ) + + def __eq__(self, other): + if not isinstance(other, OtherName): + return NotImplemented + + return self.type_id == other.type_id and self.value == other.value + + def __ne__(self, other): + return not self == other + + def __hash__(self): + return hash((self.type_id, self.value)) diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/x509/name.py b/WebCrawler/venv/Lib/site-packages/cryptography/x509/name.py new file mode 100644 index 0000000..0be876a --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/cryptography/x509/name.py @@ -0,0 +1,261 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function + +from enum import Enum + +import six + +from cryptography import utils +from cryptography.hazmat.backends import _get_backend +from cryptography.x509.oid import NameOID, ObjectIdentifier + + +class _ASN1Type(Enum): + UTF8String = 12 + NumericString = 18 + PrintableString = 19 + T61String = 20 + IA5String = 22 + UTCTime = 23 + GeneralizedTime = 24 + VisibleString = 26 + UniversalString = 28 + BMPString = 30 + + +_ASN1_TYPE_TO_ENUM = {i.value: i for i in _ASN1Type} +_SENTINEL = object() +_NAMEOID_DEFAULT_TYPE = { + NameOID.COUNTRY_NAME: _ASN1Type.PrintableString, + NameOID.JURISDICTION_COUNTRY_NAME: _ASN1Type.PrintableString, + NameOID.SERIAL_NUMBER: _ASN1Type.PrintableString, + NameOID.DN_QUALIFIER: _ASN1Type.PrintableString, + NameOID.EMAIL_ADDRESS: _ASN1Type.IA5String, + NameOID.DOMAIN_COMPONENT: _ASN1Type.IA5String, +} + +#: Short attribute names from RFC 4514: +#: https://tools.ietf.org/html/rfc4514#page-7 +_NAMEOID_TO_NAME = { + NameOID.COMMON_NAME: "CN", + NameOID.LOCALITY_NAME: "L", + NameOID.STATE_OR_PROVINCE_NAME: "ST", + NameOID.ORGANIZATION_NAME: "O", + NameOID.ORGANIZATIONAL_UNIT_NAME: "OU", + NameOID.COUNTRY_NAME: "C", + NameOID.STREET_ADDRESS: "STREET", + NameOID.DOMAIN_COMPONENT: "DC", + NameOID.USER_ID: "UID", +} + + +def _escape_dn_value(val): + """Escape special characters in RFC4514 Distinguished Name value.""" + + if not val: + return "" + + # See https://tools.ietf.org/html/rfc4514#section-2.4 + val = val.replace("\\", "\\\\") + val = val.replace('"', '\\"') + val = val.replace("+", "\\+") + val = val.replace(",", "\\,") + val = val.replace(";", "\\;") + val = val.replace("<", "\\<") + val = val.replace(">", "\\>") + val = val.replace("\0", "\\00") + + if val[0] in ("#", " "): + val = "\\" + val + if val[-1] == " ": + val = val[:-1] + "\\ " + + return val + + +class NameAttribute(object): + def __init__(self, oid, value, _type=_SENTINEL): + if not isinstance(oid, ObjectIdentifier): + raise TypeError( + "oid argument must be an ObjectIdentifier instance." + ) + + if not isinstance(value, six.text_type): + raise TypeError("value argument must be a text type.") + + if ( + oid == NameOID.COUNTRY_NAME + or oid == NameOID.JURISDICTION_COUNTRY_NAME + ): + if len(value.encode("utf8")) != 2: + raise ValueError( + "Country name must be a 2 character country code" + ) + + # The appropriate ASN1 string type varies by OID and is defined across + # multiple RFCs including 2459, 3280, and 5280. In general UTF8String + # is preferred (2459), but 3280 and 5280 specify several OIDs with + # alternate types. This means when we see the sentinel value we need + # to look up whether the OID has a non-UTF8 type. If it does, set it + # to that. Otherwise, UTF8! + if _type == _SENTINEL: + _type = _NAMEOID_DEFAULT_TYPE.get(oid, _ASN1Type.UTF8String) + + if not isinstance(_type, _ASN1Type): + raise TypeError("_type must be from the _ASN1Type enum") + + self._oid = oid + self._value = value + self._type = _type + + oid = utils.read_only_property("_oid") + value = utils.read_only_property("_value") + + def rfc4514_string(self): + """ + Format as RFC4514 Distinguished Name string. + + Use short attribute name if available, otherwise fall back to OID + dotted string. + """ + key = _NAMEOID_TO_NAME.get(self.oid, self.oid.dotted_string) + return "%s=%s" % (key, _escape_dn_value(self.value)) + + def __eq__(self, other): + if not isinstance(other, NameAttribute): + return NotImplemented + + return self.oid == other.oid and self.value == other.value + + def __ne__(self, other): + return not self == other + + def __hash__(self): + return hash((self.oid, self.value)) + + def __repr__(self): + return "".format(self) + + +class RelativeDistinguishedName(object): + def __init__(self, attributes): + attributes = list(attributes) + if not attributes: + raise ValueError("a relative distinguished name cannot be empty") + if not all(isinstance(x, NameAttribute) for x in attributes): + raise TypeError("attributes must be an iterable of NameAttribute") + + # Keep list and frozenset to preserve attribute order where it matters + self._attributes = attributes + self._attribute_set = frozenset(attributes) + + if len(self._attribute_set) != len(attributes): + raise ValueError("duplicate attributes are not allowed") + + def get_attributes_for_oid(self, oid): + return [i for i in self if i.oid == oid] + + def rfc4514_string(self): + """ + Format as RFC4514 Distinguished Name string. + + Within each RDN, attributes are joined by '+', although that is rarely + used in certificates. + """ + return "+".join(attr.rfc4514_string() for attr in self._attributes) + + def __eq__(self, other): + if not isinstance(other, RelativeDistinguishedName): + return NotImplemented + + return self._attribute_set == other._attribute_set + + def __ne__(self, other): + return not self == other + + def __hash__(self): + return hash(self._attribute_set) + + def __iter__(self): + return iter(self._attributes) + + def __len__(self): + return len(self._attributes) + + def __repr__(self): + return "".format(self.rfc4514_string()) + + +class Name(object): + def __init__(self, attributes): + attributes = list(attributes) + if all(isinstance(x, NameAttribute) for x in attributes): + self._attributes = [ + RelativeDistinguishedName([x]) for x in attributes + ] + elif all(isinstance(x, RelativeDistinguishedName) for x in attributes): + self._attributes = attributes + else: + raise TypeError( + "attributes must be a list of NameAttribute" + " or a list RelativeDistinguishedName" + ) + + def rfc4514_string(self): + """ + Format as RFC4514 Distinguished Name string. + For example 'CN=foobar.com,O=Foo Corp,C=US' + + An X.509 name is a two-level structure: a list of sets of attributes. + Each list element is separated by ',' and within each list element, set + elements are separated by '+'. The latter is almost never used in + real world certificates. According to RFC4514 section 2.1 the + RDNSequence must be reversed when converting to string representation. + """ + return ",".join( + attr.rfc4514_string() for attr in reversed(self._attributes) + ) + + def get_attributes_for_oid(self, oid): + return [i for i in self if i.oid == oid] + + @property + def rdns(self): + return self._attributes + + def public_bytes(self, backend=None): + backend = _get_backend(backend) + return backend.x509_name_bytes(self) + + def __eq__(self, other): + if not isinstance(other, Name): + return NotImplemented + + return self._attributes == other._attributes + + def __ne__(self, other): + return not self == other + + def __hash__(self): + # TODO: this is relatively expensive, if this looks like a bottleneck + # for you, consider optimizing! + return hash(tuple(self._attributes)) + + def __iter__(self): + for rdn in self._attributes: + for ava in rdn: + yield ava + + def __len__(self): + return sum(len(rdn) for rdn in self._attributes) + + def __repr__(self): + rdns = ",".join(attr.rfc4514_string() for attr in self._attributes) + + if six.PY2: + return "".format(rdns.encode("utf8")) + else: + return "".format(rdns) diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/x509/ocsp.py b/WebCrawler/venv/Lib/site-packages/cryptography/x509/ocsp.py new file mode 100644 index 0000000..f8e2722 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/cryptography/x509/ocsp.py @@ -0,0 +1,467 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function + +import abc +import datetime +from enum import Enum + +import six + +from cryptography import x509 +from cryptography.hazmat.primitives import hashes +from cryptography.x509.base import ( + _EARLIEST_UTC_TIME, + _convert_to_naive_utc_time, + _reject_duplicate_extension, +) + + +_OIDS_TO_HASH = { + "1.3.14.3.2.26": hashes.SHA1(), + "2.16.840.1.101.3.4.2.4": hashes.SHA224(), + "2.16.840.1.101.3.4.2.1": hashes.SHA256(), + "2.16.840.1.101.3.4.2.2": hashes.SHA384(), + "2.16.840.1.101.3.4.2.3": hashes.SHA512(), +} + + +class OCSPResponderEncoding(Enum): + HASH = "By Hash" + NAME = "By Name" + + +class OCSPResponseStatus(Enum): + SUCCESSFUL = 0 + MALFORMED_REQUEST = 1 + INTERNAL_ERROR = 2 + TRY_LATER = 3 + SIG_REQUIRED = 5 + UNAUTHORIZED = 6 + + +_RESPONSE_STATUS_TO_ENUM = {x.value: x for x in OCSPResponseStatus} +_ALLOWED_HASHES = ( + hashes.SHA1, + hashes.SHA224, + hashes.SHA256, + hashes.SHA384, + hashes.SHA512, +) + + +def _verify_algorithm(algorithm): + if not isinstance(algorithm, _ALLOWED_HASHES): + raise ValueError( + "Algorithm must be SHA1, SHA224, SHA256, SHA384, or SHA512" + ) + + +class OCSPCertStatus(Enum): + GOOD = 0 + REVOKED = 1 + UNKNOWN = 2 + + +_CERT_STATUS_TO_ENUM = {x.value: x for x in OCSPCertStatus} + + +def load_der_ocsp_request(data): + from cryptography.hazmat.backends.openssl.backend import backend + + return backend.load_der_ocsp_request(data) + + +def load_der_ocsp_response(data): + from cryptography.hazmat.backends.openssl.backend import backend + + return backend.load_der_ocsp_response(data) + + +class OCSPRequestBuilder(object): + def __init__(self, request=None, extensions=[]): + self._request = request + self._extensions = extensions + + def add_certificate(self, cert, issuer, algorithm): + if self._request is not None: + raise ValueError("Only one certificate can be added to a request") + + _verify_algorithm(algorithm) + if not isinstance(cert, x509.Certificate) or not isinstance( + issuer, x509.Certificate + ): + raise TypeError("cert and issuer must be a Certificate") + + return OCSPRequestBuilder((cert, issuer, algorithm), self._extensions) + + def add_extension(self, extension, critical): + if not isinstance(extension, x509.ExtensionType): + raise TypeError("extension must be an ExtensionType") + + extension = x509.Extension(extension.oid, critical, extension) + _reject_duplicate_extension(extension, self._extensions) + + return OCSPRequestBuilder( + self._request, self._extensions + [extension] + ) + + def build(self): + from cryptography.hazmat.backends.openssl.backend import backend + + if self._request is None: + raise ValueError("You must add a certificate before building") + + return backend.create_ocsp_request(self) + + +class _SingleResponse(object): + def __init__( + self, + cert, + issuer, + algorithm, + cert_status, + this_update, + next_update, + revocation_time, + revocation_reason, + ): + if not isinstance(cert, x509.Certificate) or not isinstance( + issuer, x509.Certificate + ): + raise TypeError("cert and issuer must be a Certificate") + + _verify_algorithm(algorithm) + if not isinstance(this_update, datetime.datetime): + raise TypeError("this_update must be a datetime object") + if next_update is not None and not isinstance( + next_update, datetime.datetime + ): + raise TypeError("next_update must be a datetime object or None") + + self._cert = cert + self._issuer = issuer + self._algorithm = algorithm + self._this_update = this_update + self._next_update = next_update + + if not isinstance(cert_status, OCSPCertStatus): + raise TypeError( + "cert_status must be an item from the OCSPCertStatus enum" + ) + if cert_status is not OCSPCertStatus.REVOKED: + if revocation_time is not None: + raise ValueError( + "revocation_time can only be provided if the certificate " + "is revoked" + ) + if revocation_reason is not None: + raise ValueError( + "revocation_reason can only be provided if the certificate" + " is revoked" + ) + else: + if not isinstance(revocation_time, datetime.datetime): + raise TypeError("revocation_time must be a datetime object") + + revocation_time = _convert_to_naive_utc_time(revocation_time) + if revocation_time < _EARLIEST_UTC_TIME: + raise ValueError( + "The revocation_time must be on or after" + " 1950 January 1." + ) + + if revocation_reason is not None and not isinstance( + revocation_reason, x509.ReasonFlags + ): + raise TypeError( + "revocation_reason must be an item from the ReasonFlags " + "enum or None" + ) + + self._cert_status = cert_status + self._revocation_time = revocation_time + self._revocation_reason = revocation_reason + + +class OCSPResponseBuilder(object): + def __init__( + self, response=None, responder_id=None, certs=None, extensions=[] + ): + self._response = response + self._responder_id = responder_id + self._certs = certs + self._extensions = extensions + + def add_response( + self, + cert, + issuer, + algorithm, + cert_status, + this_update, + next_update, + revocation_time, + revocation_reason, + ): + if self._response is not None: + raise ValueError("Only one response per OCSPResponse.") + + singleresp = _SingleResponse( + cert, + issuer, + algorithm, + cert_status, + this_update, + next_update, + revocation_time, + revocation_reason, + ) + return OCSPResponseBuilder( + singleresp, + self._responder_id, + self._certs, + self._extensions, + ) + + def responder_id(self, encoding, responder_cert): + if self._responder_id is not None: + raise ValueError("responder_id can only be set once") + if not isinstance(responder_cert, x509.Certificate): + raise TypeError("responder_cert must be a Certificate") + if not isinstance(encoding, OCSPResponderEncoding): + raise TypeError( + "encoding must be an element from OCSPResponderEncoding" + ) + + return OCSPResponseBuilder( + self._response, + (responder_cert, encoding), + self._certs, + self._extensions, + ) + + def certificates(self, certs): + if self._certs is not None: + raise ValueError("certificates may only be set once") + certs = list(certs) + if len(certs) == 0: + raise ValueError("certs must not be an empty list") + if not all(isinstance(x, x509.Certificate) for x in certs): + raise TypeError("certs must be a list of Certificates") + return OCSPResponseBuilder( + self._response, + self._responder_id, + certs, + self._extensions, + ) + + def add_extension(self, extension, critical): + if not isinstance(extension, x509.ExtensionType): + raise TypeError("extension must be an ExtensionType") + + extension = x509.Extension(extension.oid, critical, extension) + _reject_duplicate_extension(extension, self._extensions) + + return OCSPResponseBuilder( + self._response, + self._responder_id, + self._certs, + self._extensions + [extension], + ) + + def sign(self, private_key, algorithm): + from cryptography.hazmat.backends.openssl.backend import backend + + if self._response is None: + raise ValueError("You must add a response before signing") + if self._responder_id is None: + raise ValueError("You must add a responder_id before signing") + + return backend.create_ocsp_response( + OCSPResponseStatus.SUCCESSFUL, self, private_key, algorithm + ) + + @classmethod + def build_unsuccessful(cls, response_status): + from cryptography.hazmat.backends.openssl.backend import backend + + if not isinstance(response_status, OCSPResponseStatus): + raise TypeError( + "response_status must be an item from OCSPResponseStatus" + ) + if response_status is OCSPResponseStatus.SUCCESSFUL: + raise ValueError("response_status cannot be SUCCESSFUL") + + return backend.create_ocsp_response(response_status, None, None, None) + + +@six.add_metaclass(abc.ABCMeta) +class OCSPRequest(object): + @abc.abstractproperty + def issuer_key_hash(self): + """ + The hash of the issuer public key + """ + + @abc.abstractproperty + def issuer_name_hash(self): + """ + The hash of the issuer name + """ + + @abc.abstractproperty + def hash_algorithm(self): + """ + The hash algorithm used in the issuer name and key hashes + """ + + @abc.abstractproperty + def serial_number(self): + """ + The serial number of the cert whose status is being checked + """ + + @abc.abstractmethod + def public_bytes(self, encoding): + """ + Serializes the request to DER + """ + + @abc.abstractproperty + def extensions(self): + """ + The list of request extensions. Not single request extensions. + """ + + +@six.add_metaclass(abc.ABCMeta) +class OCSPResponse(object): + @abc.abstractproperty + def response_status(self): + """ + The status of the response. This is a value from the OCSPResponseStatus + enumeration + """ + + @abc.abstractproperty + def signature_algorithm_oid(self): + """ + The ObjectIdentifier of the signature algorithm + """ + + @abc.abstractproperty + def signature_hash_algorithm(self): + """ + Returns a HashAlgorithm corresponding to the type of the digest signed + """ + + @abc.abstractproperty + def signature(self): + """ + The signature bytes + """ + + @abc.abstractproperty + def tbs_response_bytes(self): + """ + The tbsResponseData bytes + """ + + @abc.abstractproperty + def certificates(self): + """ + A list of certificates used to help build a chain to verify the OCSP + response. This situation occurs when the OCSP responder uses a delegate + certificate. + """ + + @abc.abstractproperty + def responder_key_hash(self): + """ + The responder's key hash or None + """ + + @abc.abstractproperty + def responder_name(self): + """ + The responder's Name or None + """ + + @abc.abstractproperty + def produced_at(self): + """ + The time the response was produced + """ + + @abc.abstractproperty + def certificate_status(self): + """ + The status of the certificate (an element from the OCSPCertStatus enum) + """ + + @abc.abstractproperty + def revocation_time(self): + """ + The date of when the certificate was revoked or None if not + revoked. + """ + + @abc.abstractproperty + def revocation_reason(self): + """ + The reason the certificate was revoked or None if not specified or + not revoked. + """ + + @abc.abstractproperty + def this_update(self): + """ + The most recent time at which the status being indicated is known by + the responder to have been correct + """ + + @abc.abstractproperty + def next_update(self): + """ + The time when newer information will be available + """ + + @abc.abstractproperty + def issuer_key_hash(self): + """ + The hash of the issuer public key + """ + + @abc.abstractproperty + def issuer_name_hash(self): + """ + The hash of the issuer name + """ + + @abc.abstractproperty + def hash_algorithm(self): + """ + The hash algorithm used in the issuer name and key hashes + """ + + @abc.abstractproperty + def serial_number(self): + """ + The serial number of the cert whose status is being checked + """ + + @abc.abstractproperty + def extensions(self): + """ + The list of response extensions. Not single response extensions. + """ + + @abc.abstractproperty + def single_extensions(self): + """ + The list of single response extensions. Not response extensions. + """ diff --git a/WebCrawler/venv/Lib/site-packages/cryptography/x509/oid.py b/WebCrawler/venv/Lib/site-packages/cryptography/x509/oid.py new file mode 100644 index 0000000..2bf606e --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/cryptography/x509/oid.py @@ -0,0 +1,265 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function + +from cryptography.hazmat._oid import ObjectIdentifier +from cryptography.hazmat.primitives import hashes + + +class ExtensionOID(object): + SUBJECT_DIRECTORY_ATTRIBUTES = ObjectIdentifier("2.5.29.9") + SUBJECT_KEY_IDENTIFIER = ObjectIdentifier("2.5.29.14") + KEY_USAGE = ObjectIdentifier("2.5.29.15") + SUBJECT_ALTERNATIVE_NAME = ObjectIdentifier("2.5.29.17") + ISSUER_ALTERNATIVE_NAME = ObjectIdentifier("2.5.29.18") + BASIC_CONSTRAINTS = ObjectIdentifier("2.5.29.19") + NAME_CONSTRAINTS = ObjectIdentifier("2.5.29.30") + CRL_DISTRIBUTION_POINTS = ObjectIdentifier("2.5.29.31") + CERTIFICATE_POLICIES = ObjectIdentifier("2.5.29.32") + POLICY_MAPPINGS = ObjectIdentifier("2.5.29.33") + AUTHORITY_KEY_IDENTIFIER = ObjectIdentifier("2.5.29.35") + POLICY_CONSTRAINTS = ObjectIdentifier("2.5.29.36") + EXTENDED_KEY_USAGE = ObjectIdentifier("2.5.29.37") + FRESHEST_CRL = ObjectIdentifier("2.5.29.46") + INHIBIT_ANY_POLICY = ObjectIdentifier("2.5.29.54") + ISSUING_DISTRIBUTION_POINT = ObjectIdentifier("2.5.29.28") + AUTHORITY_INFORMATION_ACCESS = ObjectIdentifier("1.3.6.1.5.5.7.1.1") + SUBJECT_INFORMATION_ACCESS = ObjectIdentifier("1.3.6.1.5.5.7.1.11") + OCSP_NO_CHECK = ObjectIdentifier("1.3.6.1.5.5.7.48.1.5") + TLS_FEATURE = ObjectIdentifier("1.3.6.1.5.5.7.1.24") + CRL_NUMBER = ObjectIdentifier("2.5.29.20") + DELTA_CRL_INDICATOR = ObjectIdentifier("2.5.29.27") + PRECERT_SIGNED_CERTIFICATE_TIMESTAMPS = ObjectIdentifier( + "1.3.6.1.4.1.11129.2.4.2" + ) + PRECERT_POISON = ObjectIdentifier("1.3.6.1.4.1.11129.2.4.3") + SIGNED_CERTIFICATE_TIMESTAMPS = ObjectIdentifier("1.3.6.1.4.1.11129.2.4.5") + + +class OCSPExtensionOID(object): + NONCE = ObjectIdentifier("1.3.6.1.5.5.7.48.1.2") + + +class CRLEntryExtensionOID(object): + CERTIFICATE_ISSUER = ObjectIdentifier("2.5.29.29") + CRL_REASON = ObjectIdentifier("2.5.29.21") + INVALIDITY_DATE = ObjectIdentifier("2.5.29.24") + + +class NameOID(object): + COMMON_NAME = ObjectIdentifier("2.5.4.3") + COUNTRY_NAME = ObjectIdentifier("2.5.4.6") + LOCALITY_NAME = ObjectIdentifier("2.5.4.7") + STATE_OR_PROVINCE_NAME = ObjectIdentifier("2.5.4.8") + STREET_ADDRESS = ObjectIdentifier("2.5.4.9") + ORGANIZATION_NAME = ObjectIdentifier("2.5.4.10") + ORGANIZATIONAL_UNIT_NAME = ObjectIdentifier("2.5.4.11") + SERIAL_NUMBER = ObjectIdentifier("2.5.4.5") + SURNAME = ObjectIdentifier("2.5.4.4") + GIVEN_NAME = ObjectIdentifier("2.5.4.42") + TITLE = ObjectIdentifier("2.5.4.12") + GENERATION_QUALIFIER = ObjectIdentifier("2.5.4.44") + X500_UNIQUE_IDENTIFIER = ObjectIdentifier("2.5.4.45") + DN_QUALIFIER = ObjectIdentifier("2.5.4.46") + PSEUDONYM = ObjectIdentifier("2.5.4.65") + USER_ID = ObjectIdentifier("0.9.2342.19200300.100.1.1") + DOMAIN_COMPONENT = ObjectIdentifier("0.9.2342.19200300.100.1.25") + EMAIL_ADDRESS = ObjectIdentifier("1.2.840.113549.1.9.1") + JURISDICTION_COUNTRY_NAME = ObjectIdentifier("1.3.6.1.4.1.311.60.2.1.3") + JURISDICTION_LOCALITY_NAME = ObjectIdentifier("1.3.6.1.4.1.311.60.2.1.1") + JURISDICTION_STATE_OR_PROVINCE_NAME = ObjectIdentifier( + "1.3.6.1.4.1.311.60.2.1.2" + ) + BUSINESS_CATEGORY = ObjectIdentifier("2.5.4.15") + POSTAL_ADDRESS = ObjectIdentifier("2.5.4.16") + POSTAL_CODE = ObjectIdentifier("2.5.4.17") + INN = ObjectIdentifier("1.2.643.3.131.1.1") + OGRN = ObjectIdentifier("1.2.643.100.1") + SNILS = ObjectIdentifier("1.2.643.100.3") + UNSTRUCTURED_NAME = ObjectIdentifier("1.2.840.113549.1.9.2") + + +class SignatureAlgorithmOID(object): + RSA_WITH_MD5 = ObjectIdentifier("1.2.840.113549.1.1.4") + RSA_WITH_SHA1 = ObjectIdentifier("1.2.840.113549.1.1.5") + # This is an alternate OID for RSA with SHA1 that is occasionally seen + _RSA_WITH_SHA1 = ObjectIdentifier("1.3.14.3.2.29") + RSA_WITH_SHA224 = ObjectIdentifier("1.2.840.113549.1.1.14") + RSA_WITH_SHA256 = ObjectIdentifier("1.2.840.113549.1.1.11") + RSA_WITH_SHA384 = ObjectIdentifier("1.2.840.113549.1.1.12") + RSA_WITH_SHA512 = ObjectIdentifier("1.2.840.113549.1.1.13") + RSASSA_PSS = ObjectIdentifier("1.2.840.113549.1.1.10") + ECDSA_WITH_SHA1 = ObjectIdentifier("1.2.840.10045.4.1") + ECDSA_WITH_SHA224 = ObjectIdentifier("1.2.840.10045.4.3.1") + ECDSA_WITH_SHA256 = ObjectIdentifier("1.2.840.10045.4.3.2") + ECDSA_WITH_SHA384 = ObjectIdentifier("1.2.840.10045.4.3.3") + ECDSA_WITH_SHA512 = ObjectIdentifier("1.2.840.10045.4.3.4") + DSA_WITH_SHA1 = ObjectIdentifier("1.2.840.10040.4.3") + DSA_WITH_SHA224 = ObjectIdentifier("2.16.840.1.101.3.4.3.1") + DSA_WITH_SHA256 = ObjectIdentifier("2.16.840.1.101.3.4.3.2") + ED25519 = ObjectIdentifier("1.3.101.112") + ED448 = ObjectIdentifier("1.3.101.113") + GOSTR3411_94_WITH_3410_2001 = ObjectIdentifier("1.2.643.2.2.3") + GOSTR3410_2012_WITH_3411_2012_256 = ObjectIdentifier("1.2.643.7.1.1.3.2") + GOSTR3410_2012_WITH_3411_2012_512 = ObjectIdentifier("1.2.643.7.1.1.3.3") + + +_SIG_OIDS_TO_HASH = { + SignatureAlgorithmOID.RSA_WITH_MD5: hashes.MD5(), + SignatureAlgorithmOID.RSA_WITH_SHA1: hashes.SHA1(), + SignatureAlgorithmOID._RSA_WITH_SHA1: hashes.SHA1(), + SignatureAlgorithmOID.RSA_WITH_SHA224: hashes.SHA224(), + SignatureAlgorithmOID.RSA_WITH_SHA256: hashes.SHA256(), + SignatureAlgorithmOID.RSA_WITH_SHA384: hashes.SHA384(), + SignatureAlgorithmOID.RSA_WITH_SHA512: hashes.SHA512(), + SignatureAlgorithmOID.ECDSA_WITH_SHA1: hashes.SHA1(), + SignatureAlgorithmOID.ECDSA_WITH_SHA224: hashes.SHA224(), + SignatureAlgorithmOID.ECDSA_WITH_SHA256: hashes.SHA256(), + SignatureAlgorithmOID.ECDSA_WITH_SHA384: hashes.SHA384(), + SignatureAlgorithmOID.ECDSA_WITH_SHA512: hashes.SHA512(), + SignatureAlgorithmOID.DSA_WITH_SHA1: hashes.SHA1(), + SignatureAlgorithmOID.DSA_WITH_SHA224: hashes.SHA224(), + SignatureAlgorithmOID.DSA_WITH_SHA256: hashes.SHA256(), + SignatureAlgorithmOID.ED25519: None, + SignatureAlgorithmOID.ED448: None, + SignatureAlgorithmOID.GOSTR3411_94_WITH_3410_2001: None, + SignatureAlgorithmOID.GOSTR3410_2012_WITH_3411_2012_256: None, + SignatureAlgorithmOID.GOSTR3410_2012_WITH_3411_2012_512: None, +} + + +class ExtendedKeyUsageOID(object): + SERVER_AUTH = ObjectIdentifier("1.3.6.1.5.5.7.3.1") + CLIENT_AUTH = ObjectIdentifier("1.3.6.1.5.5.7.3.2") + CODE_SIGNING = ObjectIdentifier("1.3.6.1.5.5.7.3.3") + EMAIL_PROTECTION = ObjectIdentifier("1.3.6.1.5.5.7.3.4") + TIME_STAMPING = ObjectIdentifier("1.3.6.1.5.5.7.3.8") + OCSP_SIGNING = ObjectIdentifier("1.3.6.1.5.5.7.3.9") + ANY_EXTENDED_KEY_USAGE = ObjectIdentifier("2.5.29.37.0") + + +class AuthorityInformationAccessOID(object): + CA_ISSUERS = ObjectIdentifier("1.3.6.1.5.5.7.48.2") + OCSP = ObjectIdentifier("1.3.6.1.5.5.7.48.1") + + +class SubjectInformationAccessOID(object): + CA_REPOSITORY = ObjectIdentifier("1.3.6.1.5.5.7.48.5") + + +class CertificatePoliciesOID(object): + CPS_QUALIFIER = ObjectIdentifier("1.3.6.1.5.5.7.2.1") + CPS_USER_NOTICE = ObjectIdentifier("1.3.6.1.5.5.7.2.2") + ANY_POLICY = ObjectIdentifier("2.5.29.32.0") + + +class AttributeOID(object): + CHALLENGE_PASSWORD = ObjectIdentifier("1.2.840.113549.1.9.7") + UNSTRUCTURED_NAME = ObjectIdentifier("1.2.840.113549.1.9.2") + + +_OID_NAMES = { + NameOID.COMMON_NAME: "commonName", + NameOID.COUNTRY_NAME: "countryName", + NameOID.LOCALITY_NAME: "localityName", + NameOID.STATE_OR_PROVINCE_NAME: "stateOrProvinceName", + NameOID.STREET_ADDRESS: "streetAddress", + NameOID.ORGANIZATION_NAME: "organizationName", + NameOID.ORGANIZATIONAL_UNIT_NAME: "organizationalUnitName", + NameOID.SERIAL_NUMBER: "serialNumber", + NameOID.SURNAME: "surname", + NameOID.GIVEN_NAME: "givenName", + NameOID.TITLE: "title", + NameOID.GENERATION_QUALIFIER: "generationQualifier", + NameOID.X500_UNIQUE_IDENTIFIER: "x500UniqueIdentifier", + NameOID.DN_QUALIFIER: "dnQualifier", + NameOID.PSEUDONYM: "pseudonym", + NameOID.USER_ID: "userID", + NameOID.DOMAIN_COMPONENT: "domainComponent", + NameOID.EMAIL_ADDRESS: "emailAddress", + NameOID.JURISDICTION_COUNTRY_NAME: "jurisdictionCountryName", + NameOID.JURISDICTION_LOCALITY_NAME: "jurisdictionLocalityName", + NameOID.JURISDICTION_STATE_OR_PROVINCE_NAME: ( + "jurisdictionStateOrProvinceName" + ), + NameOID.BUSINESS_CATEGORY: "businessCategory", + NameOID.POSTAL_ADDRESS: "postalAddress", + NameOID.POSTAL_CODE: "postalCode", + NameOID.INN: "INN", + NameOID.OGRN: "OGRN", + NameOID.SNILS: "SNILS", + NameOID.UNSTRUCTURED_NAME: "unstructuredName", + SignatureAlgorithmOID.RSA_WITH_MD5: "md5WithRSAEncryption", + SignatureAlgorithmOID.RSA_WITH_SHA1: "sha1WithRSAEncryption", + SignatureAlgorithmOID.RSA_WITH_SHA224: "sha224WithRSAEncryption", + SignatureAlgorithmOID.RSA_WITH_SHA256: "sha256WithRSAEncryption", + SignatureAlgorithmOID.RSA_WITH_SHA384: "sha384WithRSAEncryption", + SignatureAlgorithmOID.RSA_WITH_SHA512: "sha512WithRSAEncryption", + SignatureAlgorithmOID.RSASSA_PSS: "RSASSA-PSS", + SignatureAlgorithmOID.ECDSA_WITH_SHA1: "ecdsa-with-SHA1", + SignatureAlgorithmOID.ECDSA_WITH_SHA224: "ecdsa-with-SHA224", + SignatureAlgorithmOID.ECDSA_WITH_SHA256: "ecdsa-with-SHA256", + SignatureAlgorithmOID.ECDSA_WITH_SHA384: "ecdsa-with-SHA384", + SignatureAlgorithmOID.ECDSA_WITH_SHA512: "ecdsa-with-SHA512", + SignatureAlgorithmOID.DSA_WITH_SHA1: "dsa-with-sha1", + SignatureAlgorithmOID.DSA_WITH_SHA224: "dsa-with-sha224", + SignatureAlgorithmOID.DSA_WITH_SHA256: "dsa-with-sha256", + SignatureAlgorithmOID.ED25519: "ed25519", + SignatureAlgorithmOID.ED448: "ed448", + SignatureAlgorithmOID.GOSTR3411_94_WITH_3410_2001: ( + "GOST R 34.11-94 with GOST R 34.10-2001" + ), + SignatureAlgorithmOID.GOSTR3410_2012_WITH_3411_2012_256: ( + "GOST R 34.10-2012 with GOST R 34.11-2012 (256 bit)" + ), + SignatureAlgorithmOID.GOSTR3410_2012_WITH_3411_2012_512: ( + "GOST R 34.10-2012 with GOST R 34.11-2012 (512 bit)" + ), + ExtendedKeyUsageOID.SERVER_AUTH: "serverAuth", + ExtendedKeyUsageOID.CLIENT_AUTH: "clientAuth", + ExtendedKeyUsageOID.CODE_SIGNING: "codeSigning", + ExtendedKeyUsageOID.EMAIL_PROTECTION: "emailProtection", + ExtendedKeyUsageOID.TIME_STAMPING: "timeStamping", + ExtendedKeyUsageOID.OCSP_SIGNING: "OCSPSigning", + ExtensionOID.SUBJECT_DIRECTORY_ATTRIBUTES: "subjectDirectoryAttributes", + ExtensionOID.SUBJECT_KEY_IDENTIFIER: "subjectKeyIdentifier", + ExtensionOID.KEY_USAGE: "keyUsage", + ExtensionOID.SUBJECT_ALTERNATIVE_NAME: "subjectAltName", + ExtensionOID.ISSUER_ALTERNATIVE_NAME: "issuerAltName", + ExtensionOID.BASIC_CONSTRAINTS: "basicConstraints", + ExtensionOID.PRECERT_SIGNED_CERTIFICATE_TIMESTAMPS: ( + "signedCertificateTimestampList" + ), + ExtensionOID.SIGNED_CERTIFICATE_TIMESTAMPS: ( + "signedCertificateTimestampList" + ), + ExtensionOID.PRECERT_POISON: "ctPoison", + CRLEntryExtensionOID.CRL_REASON: "cRLReason", + CRLEntryExtensionOID.INVALIDITY_DATE: "invalidityDate", + CRLEntryExtensionOID.CERTIFICATE_ISSUER: "certificateIssuer", + ExtensionOID.NAME_CONSTRAINTS: "nameConstraints", + ExtensionOID.CRL_DISTRIBUTION_POINTS: "cRLDistributionPoints", + ExtensionOID.CERTIFICATE_POLICIES: "certificatePolicies", + ExtensionOID.POLICY_MAPPINGS: "policyMappings", + ExtensionOID.AUTHORITY_KEY_IDENTIFIER: "authorityKeyIdentifier", + ExtensionOID.POLICY_CONSTRAINTS: "policyConstraints", + ExtensionOID.EXTENDED_KEY_USAGE: "extendedKeyUsage", + ExtensionOID.FRESHEST_CRL: "freshestCRL", + ExtensionOID.INHIBIT_ANY_POLICY: "inhibitAnyPolicy", + ExtensionOID.ISSUING_DISTRIBUTION_POINT: ("issuingDistributionPoint"), + ExtensionOID.AUTHORITY_INFORMATION_ACCESS: "authorityInfoAccess", + ExtensionOID.SUBJECT_INFORMATION_ACCESS: "subjectInfoAccess", + ExtensionOID.OCSP_NO_CHECK: "OCSPNoCheck", + ExtensionOID.CRL_NUMBER: "cRLNumber", + ExtensionOID.DELTA_CRL_INDICATOR: "deltaCRLIndicator", + ExtensionOID.TLS_FEATURE: "TLSFeature", + AuthorityInformationAccessOID.OCSP: "OCSP", + AuthorityInformationAccessOID.CA_ISSUERS: "caIssuers", + SubjectInformationAccessOID.CA_REPOSITORY: "caRepository", + CertificatePoliciesOID.CPS_QUALIFIER: "id-qt-cps", + CertificatePoliciesOID.CPS_USER_NOTICE: "id-qt-unotice", + OCSPExtensionOID.NONCE: "OCSPNonce", + AttributeOID.CHALLENGE_PASSWORD: "challengePassword", +} diff --git a/WebCrawler/venv/Lib/site-packages/cssselect-1.1.0.dist-info/AUTHORS b/WebCrawler/venv/Lib/site-packages/cssselect-1.1.0.dist-info/AUTHORS new file mode 100644 index 0000000..66dcc22 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/cssselect-1.1.0.dist-info/AUTHORS @@ -0,0 +1,13 @@ +Daniel Graña +Ian Bicking +James Salter +Laurence Rowe +Mikhail Korobov +Nik Nyby +Paul Tremberth +Simon Potter +Simon Sapin +Stefan Behnel +Thomas Grainger +Varialus +Arthur Darcet diff --git a/WebCrawler/venv/Lib/site-packages/cssselect-1.1.0.dist-info/INSTALLER b/WebCrawler/venv/Lib/site-packages/cssselect-1.1.0.dist-info/INSTALLER new file mode 100644 index 0000000..a1b589e --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/cssselect-1.1.0.dist-info/INSTALLER @@ -0,0 +1 @@ +pip diff --git a/WebCrawler/venv/Lib/site-packages/cssselect-1.1.0.dist-info/LICENSE b/WebCrawler/venv/Lib/site-packages/cssselect-1.1.0.dist-info/LICENSE new file mode 100644 index 0000000..396e079 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/cssselect-1.1.0.dist-info/LICENSE @@ -0,0 +1,32 @@ +Copyright (c) 2007-2012 Ian Bicking and contributors. See AUTHORS +for more details. + +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + +1. Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in +the documentation and/or other materials provided with the +distribution. + +3. Neither the name of Ian Bicking nor the names of its contributors may +be used to endorse or promote products derived from this software +without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL IAN BICKING OR +CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/WebCrawler/venv/Lib/site-packages/cssselect-1.1.0.dist-info/METADATA b/WebCrawler/venv/Lib/site-packages/cssselect-1.1.0.dist-info/METADATA new file mode 100644 index 0000000..569a646 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/cssselect-1.1.0.dist-info/METADATA @@ -0,0 +1,65 @@ +Metadata-Version: 2.1 +Name: cssselect +Version: 1.1.0 +Summary: cssselect parses CSS3 Selectors and translates them to XPath 1.0 +Home-page: https://github.com/scrapy/cssselect +Author: Ian Bicking +Author-email: ianb@colorstudy.com +Maintainer: Paul Tremberth +Maintainer-email: paul.tremberth@gmail.com +License: BSD +Platform: UNKNOWN +Classifier: Development Status :: 4 - Beta +Classifier: Intended Audience :: Developers +Classifier: License :: OSI Approved :: BSD License +Classifier: Programming Language :: Python :: 2 +Classifier: Programming Language :: Python :: 2.7 +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3.4 +Classifier: Programming Language :: Python :: 3.5 +Classifier: Programming Language :: Python :: 3.6 +Requires-Python: >=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.* + +=================================== +cssselect: CSS Selectors for Python +=================================== + +.. image:: https://img.shields.io/pypi/v/cssselect.svg + :target: https://pypi.python.org/pypi/cssselect + :alt: PyPI Version + +.. image:: https://img.shields.io/pypi/pyversions/cssselect.svg + :target: https://pypi.python.org/pypi/cssselect + :alt: Supported Python Versions + +.. image:: https://img.shields.io/travis/scrapy/cssselect/master.svg + :target: https://travis-ci.org/scrapy/cssselect + :alt: Build Status + +.. image:: https://img.shields.io/codecov/c/github/scrapy/cssselect/master.svg + :target: https://codecov.io/github/scrapy/cssselect?branch=master + :alt: Coverage report + +*cssselect* parses `CSS3 Selectors`_ and translate them to `XPath 1.0`_ +expressions. Such expressions can be used in lxml_ or another XPath engine +to find the matching elements in an XML or HTML document. + +This module used to live inside of lxml as ``lxml.cssselect`` before it was +extracted as a stand-alone project. + +.. _CSS3 Selectors: https://www.w3.org/TR/css3-selectors/ +.. _XPath 1.0: https://www.w3.org/TR/xpath/ +.. _lxml: http://lxml.de/ + + +Quick facts: + +* Free software: BSD licensed +* Compatible with Python 2.7 and 3.4+ +* Latest documentation `on Read the Docs `_ +* Source, issues and pull requests `on GitHub + `_ +* Releases `on PyPI `_ +* Install with ``pip install cssselect`` + + diff --git a/WebCrawler/venv/Lib/site-packages/cssselect-1.1.0.dist-info/RECORD b/WebCrawler/venv/Lib/site-packages/cssselect-1.1.0.dist-info/RECORD new file mode 100644 index 0000000..c5fd1d4 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/cssselect-1.1.0.dist-info/RECORD @@ -0,0 +1,13 @@ +cssselect-1.1.0.dist-info/AUTHORS,sha256=0xOj3H8iII6M_KWmoDYQRuzTrYrLPbJ8K2kVSAoQ5zQ,171 +cssselect-1.1.0.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4 +cssselect-1.1.0.dist-info/LICENSE,sha256=XI2p90Tgr7qBpIybXb5zBI95izKH1vGvigXuCOuxCJI,1517 +cssselect-1.1.0.dist-info/METADATA,sha256=L1k9gxlz3quPBMPepkTOo0tPXV2l8cQl-yZGnO-gNUc,2332 +cssselect-1.1.0.dist-info/RECORD,, +cssselect-1.1.0.dist-info/WHEEL,sha256=h_aVn5OB2IERUjMbi2pucmR_zzWJtk303YXvhh60NJ8,110 +cssselect-1.1.0.dist-info/top_level.txt,sha256=GKK3Utu_ceog6CK27VXNdbyiqGnoyJo3970FLo5JdUU,10 +cssselect/__init__.py,sha256=fmdpKrz5-uYAzXUh3LMOrEPiJMENB1gf962v5Nq5yis,639 +cssselect/__pycache__/__init__.cpython-39.pyc,, +cssselect/__pycache__/parser.cpython-39.pyc,, +cssselect/__pycache__/xpath.cpython-39.pyc,, +cssselect/parser.py,sha256=jrlRxEpSFfiqYHKfUHjgERdMQwFbvsh3sONi_HiTDys,26145 +cssselect/xpath.py,sha256=Izy4CXmK6zMHg13PP63bBp065eFZNBfPJeNyhjSsS7Q,28257 diff --git a/WebCrawler/venv/Lib/site-packages/cssselect-1.1.0.dist-info/WHEEL b/WebCrawler/venv/Lib/site-packages/cssselect-1.1.0.dist-info/WHEEL new file mode 100644 index 0000000..78e6f69 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/cssselect-1.1.0.dist-info/WHEEL @@ -0,0 +1,6 @@ +Wheel-Version: 1.0 +Generator: bdist_wheel (0.33.4) +Root-Is-Purelib: true +Tag: py2-none-any +Tag: py3-none-any + diff --git a/WebCrawler/venv/Lib/site-packages/cssselect-1.1.0.dist-info/top_level.txt b/WebCrawler/venv/Lib/site-packages/cssselect-1.1.0.dist-info/top_level.txt new file mode 100644 index 0000000..d2a154e --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/cssselect-1.1.0.dist-info/top_level.txt @@ -0,0 +1 @@ +cssselect diff --git a/WebCrawler/venv/Lib/site-packages/cssselect/__init__.py b/WebCrawler/venv/Lib/site-packages/cssselect/__init__.py new file mode 100644 index 0000000..b41cef9 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/cssselect/__init__.py @@ -0,0 +1,22 @@ +# -*- coding: utf-8 -*- +""" + CSS Selectors based on XPath + ============================ + + This module supports selecting XML/HTML elements based on CSS selectors. + See the `CSSSelector` class for details. + + + :copyright: (c) 2007-2012 Ian Bicking and contributors. + See AUTHORS for more details. + :license: BSD, see LICENSE for more details. + +""" + +from cssselect.parser import (parse, Selector, FunctionalPseudoElement, + SelectorError, SelectorSyntaxError) +from cssselect.xpath import GenericTranslator, HTMLTranslator, ExpressionError + + +VERSION = '1.1.0' +__version__ = VERSION diff --git a/WebCrawler/venv/Lib/site-packages/cssselect/__pycache__/__init__.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/cssselect/__pycache__/__init__.cpython-39.pyc new file mode 100644 index 0000000..fbd7b80 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/cssselect/__pycache__/__init__.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/cssselect/__pycache__/parser.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/cssselect/__pycache__/parser.cpython-39.pyc new file mode 100644 index 0000000..264724c Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/cssselect/__pycache__/parser.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/cssselect/__pycache__/xpath.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/cssselect/__pycache__/xpath.cpython-39.pyc new file mode 100644 index 0000000..4922a96 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/cssselect/__pycache__/xpath.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/cssselect/parser.py b/WebCrawler/venv/Lib/site-packages/cssselect/parser.py new file mode 100644 index 0000000..7125030 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/cssselect/parser.py @@ -0,0 +1,835 @@ +# -*- coding: utf-8 -*- +""" + cssselect.parser + ================ + + Tokenizer, parser and parsed objects for CSS selectors. + + + :copyright: (c) 2007-2012 Ian Bicking and contributors. + See AUTHORS for more details. + :license: BSD, see LICENSE for more details. + +""" + +import sys +import re +import operator + + +if sys.version_info[0] < 3: + _unicode = unicode + _unichr = unichr +else: + _unicode = str + _unichr = chr + + +def ascii_lower(string): + """Lower-case, but only in the ASCII range.""" + return string.encode('utf8').lower().decode('utf8') + + +class SelectorError(Exception): + """Common parent for :class:`SelectorSyntaxError` and + :class:`ExpressionError`. + + You can just use ``except SelectorError:`` when calling + :meth:`~GenericTranslator.css_to_xpath` and handle both exceptions types. + + """ + +class SelectorSyntaxError(SelectorError, SyntaxError): + """Parsing a selector that does not match the grammar.""" + + +#### Parsed objects + +class Selector(object): + """ + Represents a parsed selector. + + :meth:`~GenericTranslator.selector_to_xpath` accepts this object, + but ignores :attr:`pseudo_element`. It is the user’s responsibility + to account for pseudo-elements and reject selectors with unknown + or unsupported pseudo-elements. + + """ + def __init__(self, tree, pseudo_element=None): + self.parsed_tree = tree + if pseudo_element is not None and not isinstance( + pseudo_element, FunctionalPseudoElement): + pseudo_element = ascii_lower(pseudo_element) + #: A :class:`FunctionalPseudoElement`, + #: or the identifier for the pseudo-element as a string, + # or ``None``. + #: + #: +-------------------------+----------------+--------------------------------+ + #: | | Selector | Pseudo-element | + #: +=========================+================+================================+ + #: | CSS3 syntax | ``a::before`` | ``'before'`` | + #: +-------------------------+----------------+--------------------------------+ + #: | Older syntax | ``a:before`` | ``'before'`` | + #: +-------------------------+----------------+--------------------------------+ + #: | From the Lists3_ draft, | ``li::marker`` | ``'marker'`` | + #: | not in Selectors3 | | | + #: +-------------------------+----------------+--------------------------------+ + #: | Invalid pseudo-class | ``li:marker`` | ``None`` | + #: +-------------------------+----------------+--------------------------------+ + #: | Functional | ``a::foo(2)`` | ``FunctionalPseudoElement(…)`` | + #: +-------------------------+----------------+--------------------------------+ + #: + #: .. _Lists3: http://www.w3.org/TR/2011/WD-css3-lists-20110524/#marker-pseudoelement + self.pseudo_element = pseudo_element + + def __repr__(self): + if isinstance(self.pseudo_element, FunctionalPseudoElement): + pseudo_element = repr(self.pseudo_element) + elif self.pseudo_element: + pseudo_element = '::%s' % self.pseudo_element + else: + pseudo_element = '' + return '%s[%r%s]' % ( + self.__class__.__name__, self.parsed_tree, pseudo_element) + + def canonical(self): + """Return a CSS representation for this selector (a string) + """ + if isinstance(self.pseudo_element, FunctionalPseudoElement): + pseudo_element = '::%s' % self.pseudo_element.canonical() + elif self.pseudo_element: + pseudo_element = '::%s' % self.pseudo_element + else: + pseudo_element = '' + res = '%s%s' % (self.parsed_tree.canonical(), pseudo_element) + if len(res) > 1: + res = res.lstrip('*') + return res + + def specificity(self): + """Return the specificity_ of this selector as a tuple of 3 integers. + + .. _specificity: http://www.w3.org/TR/selectors/#specificity + + """ + a, b, c = self.parsed_tree.specificity() + if self.pseudo_element: + c += 1 + return a, b, c + + +class Class(object): + """ + Represents selector.class_name + """ + def __init__(self, selector, class_name): + self.selector = selector + self.class_name = class_name + + def __repr__(self): + return '%s[%r.%s]' % ( + self.__class__.__name__, self.selector, self.class_name) + + def canonical(self): + return '%s.%s' % (self.selector.canonical(), self.class_name) + + def specificity(self): + a, b, c = self.selector.specificity() + b += 1 + return a, b, c + + +class FunctionalPseudoElement(object): + """ + Represents selector::name(arguments) + + .. attribute:: name + + The name (identifier) of the pseudo-element, as a string. + + .. attribute:: arguments + + The arguments of the pseudo-element, as a list of tokens. + + **Note:** tokens are not part of the public API, + and may change between cssselect versions. + Use at your own risks. + + """ + def __init__(self, name, arguments): + self.name = ascii_lower(name) + self.arguments = arguments + + def __repr__(self): + return '%s[::%s(%r)]' % ( + self.__class__.__name__, self.name, + [token.value for token in self.arguments]) + + def argument_types(self): + return [token.type for token in self.arguments] + + def canonical(self): + args = ''.join(token.css() for token in self.arguments) + return '%s(%s)' % (self.name, args) + + def specificity(self): + a, b, c = self.selector.specificity() + b += 1 + return a, b, c + + +class Function(object): + """ + Represents selector:name(expr) + """ + def __init__(self, selector, name, arguments): + self.selector = selector + self.name = ascii_lower(name) + self.arguments = arguments + + def __repr__(self): + return '%s[%r:%s(%r)]' % ( + self.__class__.__name__, self.selector, self.name, + [token.value for token in self.arguments]) + + def argument_types(self): + return [token.type for token in self.arguments] + + def canonical(self): + args = ''.join(token.css() for token in self.arguments) + return '%s:%s(%s)' % (self.selector.canonical(), self.name, args) + + def specificity(self): + a, b, c = self.selector.specificity() + b += 1 + return a, b, c + + +class Pseudo(object): + """ + Represents selector:ident + """ + def __init__(self, selector, ident): + self.selector = selector + self.ident = ascii_lower(ident) + + def __repr__(self): + return '%s[%r:%s]' % ( + self.__class__.__name__, self.selector, self.ident) + + def canonical(self): + return '%s:%s' % (self.selector.canonical(), self.ident) + + def specificity(self): + a, b, c = self.selector.specificity() + b += 1 + return a, b, c + + +class Negation(object): + """ + Represents selector:not(subselector) + """ + def __init__(self, selector, subselector): + self.selector = selector + self.subselector = subselector + + def __repr__(self): + return '%s[%r:not(%r)]' % ( + self.__class__.__name__, self.selector, self.subselector) + + def canonical(self): + subsel = self.subselector.canonical() + if len(subsel) > 1: + subsel = subsel.lstrip('*') + return '%s:not(%s)' % (self.selector.canonical(), subsel) + + def specificity(self): + a1, b1, c1 = self.selector.specificity() + a2, b2, c2 = self.subselector.specificity() + return a1 + a2, b1 + b2, c1 + c2 + + +class Attrib(object): + """ + Represents selector[namespace|attrib operator value] + """ + def __init__(self, selector, namespace, attrib, operator, value): + self.selector = selector + self.namespace = namespace + self.attrib = attrib + self.operator = operator + self.value = value + + def __repr__(self): + if self.namespace: + attrib = '%s|%s' % (self.namespace, self.attrib) + else: + attrib = self.attrib + if self.operator == 'exists': + return '%s[%r[%s]]' % ( + self.__class__.__name__, self.selector, attrib) + else: + return '%s[%r[%s %s %r]]' % ( + self.__class__.__name__, self.selector, attrib, + self.operator, self.value.value) + + def canonical(self): + if self.namespace: + attrib = '%s|%s' % (self.namespace, self.attrib) + else: + attrib = self.attrib + + if self.operator == 'exists': + op = attrib + else: + op = '%s%s%s' % (attrib, self.operator, self.value.css()) + + return '%s[%s]' % (self.selector.canonical(), op) + + def specificity(self): + a, b, c = self.selector.specificity() + b += 1 + return a, b, c + + +class Element(object): + """ + Represents namespace|element + + `None` is for the universal selector '*' + + """ + def __init__(self, namespace=None, element=None): + self.namespace = namespace + self.element = element + + def __repr__(self): + return '%s[%s]' % (self.__class__.__name__, self.canonical()) + + def canonical(self): + element = self.element or '*' + if self.namespace: + element = '%s|%s' % (self.namespace, element) + return element + + def specificity(self): + if self.element: + return 0, 0, 1 + else: + return 0, 0, 0 + + +class Hash(object): + """ + Represents selector#id + """ + def __init__(self, selector, id): + self.selector = selector + self.id = id + + def __repr__(self): + return '%s[%r#%s]' % ( + self.__class__.__name__, self.selector, self.id) + + def canonical(self): + return '%s#%s' % (self.selector.canonical(), self.id) + + def specificity(self): + a, b, c = self.selector.specificity() + a += 1 + return a, b, c + + +class CombinedSelector(object): + def __init__(self, selector, combinator, subselector): + assert selector is not None + self.selector = selector + self.combinator = combinator + self.subselector = subselector + + def __repr__(self): + if self.combinator == ' ': + comb = '' + else: + comb = self.combinator + return '%s[%r %s %r]' % ( + self.__class__.__name__, self.selector, comb, self.subselector) + + def canonical(self): + subsel = self.subselector.canonical() + if len(subsel) > 1: + subsel = subsel.lstrip('*') + return '%s %s %s' % ( + self.selector.canonical(), self.combinator, subsel) + + def specificity(self): + a1, b1, c1 = self.selector.specificity() + a2, b2, c2 = self.subselector.specificity() + return a1 + a2, b1 + b2, c1 + c2 + + +#### Parser + +# foo +_el_re = re.compile(r'^[ \t\r\n\f]*([a-zA-Z]+)[ \t\r\n\f]*$') + +# foo#bar or #bar +_id_re = re.compile(r'^[ \t\r\n\f]*([a-zA-Z]*)#([a-zA-Z0-9_-]+)[ \t\r\n\f]*$') + +# foo.bar or .bar +_class_re = re.compile( + r'^[ \t\r\n\f]*([a-zA-Z]*)\.([a-zA-Z][a-zA-Z0-9_-]*)[ \t\r\n\f]*$') + + +def parse(css): + """Parse a CSS *group of selectors*. + + If you don't care about pseudo-elements or selector specificity, + you can skip this and use :meth:`~GenericTranslator.css_to_xpath`. + + :param css: + A *group of selectors* as an Unicode string. + :raises: + :class:`SelectorSyntaxError` on invalid selectors. + :returns: + A list of parsed :class:`Selector` objects, one for each + selector in the comma-separated group. + + """ + # Fast path for simple cases + match = _el_re.match(css) + if match: + return [Selector(Element(element=match.group(1)))] + match = _id_re.match(css) + if match is not None: + return [Selector(Hash(Element(element=match.group(1) or None), + match.group(2)))] + match = _class_re.match(css) + if match is not None: + return [Selector(Class(Element(element=match.group(1) or None), + match.group(2)))] + + stream = TokenStream(tokenize(css)) + stream.source = css + return list(parse_selector_group(stream)) +# except SelectorSyntaxError: +# e = sys.exc_info()[1] +# message = "%s at %s -> %r" % ( +# e, stream.used, stream.peek()) +# e.msg = message +# e.args = tuple([message]) +# raise + + +def parse_selector_group(stream): + stream.skip_whitespace() + while 1: + yield Selector(*parse_selector(stream)) + if stream.peek() == ('DELIM', ','): + stream.next() + stream.skip_whitespace() + else: + break + +def parse_selector(stream): + result, pseudo_element = parse_simple_selector(stream) + while 1: + stream.skip_whitespace() + peek = stream.peek() + if peek in (('EOF', None), ('DELIM', ',')): + break + if pseudo_element: + raise SelectorSyntaxError( + 'Got pseudo-element ::%s not at the end of a selector' + % pseudo_element) + if peek.is_delim('+', '>', '~'): + # A combinator + combinator = stream.next().value + stream.skip_whitespace() + else: + # By exclusion, the last parse_simple_selector() ended + # at peek == ' ' + combinator = ' ' + next_selector, pseudo_element = parse_simple_selector(stream) + result = CombinedSelector(result, combinator, next_selector) + return result, pseudo_element + + +def parse_simple_selector(stream, inside_negation=False): + stream.skip_whitespace() + selector_start = len(stream.used) + peek = stream.peek() + if peek.type == 'IDENT' or peek == ('DELIM', '*'): + if peek.type == 'IDENT': + namespace = stream.next().value + else: + stream.next() + namespace = None + if stream.peek() == ('DELIM', '|'): + stream.next() + element = stream.next_ident_or_star() + else: + element = namespace + namespace = None + else: + element = namespace = None + result = Element(namespace, element) + pseudo_element = None + while 1: + peek = stream.peek() + if peek.type in ('S', 'EOF') or peek.is_delim(',', '+', '>', '~') or ( + inside_negation and peek == ('DELIM', ')')): + break + if pseudo_element: + raise SelectorSyntaxError( + 'Got pseudo-element ::%s not at the end of a selector' + % pseudo_element) + if peek.type == 'HASH': + result = Hash(result, stream.next().value) + elif peek == ('DELIM', '.'): + stream.next() + result = Class(result, stream.next_ident()) + elif peek == ('DELIM', '|'): + stream.next() + result = Element(None, stream.next_ident()) + elif peek == ('DELIM', '['): + stream.next() + result = parse_attrib(result, stream) + elif peek == ('DELIM', ':'): + stream.next() + if stream.peek() == ('DELIM', ':'): + stream.next() + pseudo_element = stream.next_ident() + if stream.peek() == ('DELIM', '('): + stream.next() + pseudo_element = FunctionalPseudoElement( + pseudo_element, parse_arguments(stream)) + continue + ident = stream.next_ident() + if ident.lower() in ('first-line', 'first-letter', + 'before', 'after'): + # Special case: CSS 2.1 pseudo-elements can have a single ':' + # Any new pseudo-element must have two. + pseudo_element = _unicode(ident) + continue + if stream.peek() != ('DELIM', '('): + result = Pseudo(result, ident) + if result.__repr__() == 'Pseudo[Element[*]:scope]': + if not (len(stream.used) == 2 or + (len(stream.used) == 3 + and stream.used[0].type == 'S')): + raise SelectorSyntaxError( + 'Got immediate child pseudo-element ":scope" ' + 'not at the start of a selector') + continue + stream.next() + stream.skip_whitespace() + if ident.lower() == 'not': + if inside_negation: + raise SelectorSyntaxError('Got nested :not()') + argument, argument_pseudo_element = parse_simple_selector( + stream, inside_negation=True) + next = stream.next() + if argument_pseudo_element: + raise SelectorSyntaxError( + 'Got pseudo-element ::%s inside :not() at %s' + % (argument_pseudo_element, next.pos)) + if next != ('DELIM', ')'): + raise SelectorSyntaxError("Expected ')', got %s" % (next,)) + result = Negation(result, argument) + else: + result = Function(result, ident, parse_arguments(stream)) + else: + raise SelectorSyntaxError( + "Expected selector, got %s" % (peek,)) + if len(stream.used) == selector_start: + raise SelectorSyntaxError( + "Expected selector, got %s" % (stream.peek(),)) + return result, pseudo_element + + +def parse_arguments(stream): + arguments = [] + while 1: + stream.skip_whitespace() + next = stream.next() + if next.type in ('IDENT', 'STRING', 'NUMBER') or next in [ + ('DELIM', '+'), ('DELIM', '-')]: + arguments.append(next) + elif next == ('DELIM', ')'): + return arguments + else: + raise SelectorSyntaxError( + "Expected an argument, got %s" % (next,)) + + +def parse_attrib(selector, stream): + stream.skip_whitespace() + attrib = stream.next_ident_or_star() + if attrib is None and stream.peek() != ('DELIM', '|'): + raise SelectorSyntaxError( + "Expected '|', got %s" % (stream.peek(),)) + if stream.peek() == ('DELIM', '|'): + stream.next() + if stream.peek() == ('DELIM', '='): + namespace = None + stream.next() + op = '|=' + else: + namespace = attrib + attrib = stream.next_ident() + op = None + else: + namespace = op = None + if op is None: + stream.skip_whitespace() + next = stream.next() + if next == ('DELIM', ']'): + return Attrib(selector, namespace, attrib, 'exists', None) + elif next == ('DELIM', '='): + op = '=' + elif next.is_delim('^', '$', '*', '~', '|', '!') and ( + stream.peek() == ('DELIM', '=')): + op = next.value + '=' + stream.next() + else: + raise SelectorSyntaxError( + "Operator expected, got %s" % (next,)) + stream.skip_whitespace() + value = stream.next() + if value.type not in ('IDENT', 'STRING'): + raise SelectorSyntaxError( + "Expected string or ident, got %s" % (value,)) + stream.skip_whitespace() + next = stream.next() + if next != ('DELIM', ']'): + raise SelectorSyntaxError( + "Expected ']', got %s" % (next,)) + return Attrib(selector, namespace, attrib, op, value) + + +def parse_series(tokens): + """ + Parses the arguments for :nth-child() and friends. + + :raises: A list of tokens + :returns: :``(a, b)`` + + """ + for token in tokens: + if token.type == 'STRING': + raise ValueError('String tokens not allowed in series.') + s = ''.join(token.value for token in tokens).strip() + if s == 'odd': + return 2, 1 + elif s == 'even': + return 2, 0 + elif s == 'n': + return 1, 0 + if 'n' not in s: + # Just b + return 0, int(s) + a, b = s.split('n', 1) + if not a: + a = 1 + elif a == '-' or a == '+': + a = int(a+'1') + else: + a = int(a) + if not b: + b = 0 + else: + b = int(b) + return a, b + + +#### Token objects + +class Token(tuple): + def __new__(cls, type_, value, pos): + obj = tuple.__new__(cls, (type_, value)) + obj.pos = pos + return obj + + def __repr__(self): + return "<%s '%s' at %i>" % (self.type, self.value, self.pos) + + def is_delim(self, *values): + return self.type == 'DELIM' and self.value in values + + type = property(operator.itemgetter(0)) + value = property(operator.itemgetter(1)) + + def css(self): + if self.type == 'STRING': + return repr(self.value) + else: + return self.value + + +class EOFToken(Token): + def __new__(cls, pos): + return Token.__new__(cls, 'EOF', None, pos) + + def __repr__(self): + return '<%s at %i>' % (self.type, self.pos) + + +#### Tokenizer + + +class TokenMacros: + unicode_escape = r'\\([0-9a-f]{1,6})(?:\r\n|[ \n\r\t\f])?' + escape = unicode_escape + r'|\\[^\n\r\f0-9a-f]' + string_escape = r'\\(?:\n|\r\n|\r|\f)|' + escape + nonascii = r'[^\0-\177]' + nmchar = '[_a-z0-9-]|%s|%s' % (escape, nonascii) + nmstart = '[_a-z]|%s|%s' % (escape, nonascii) + +def _compile(pattern): + return re.compile(pattern % vars(TokenMacros), re.IGNORECASE).match + +_match_whitespace = _compile(r'[ \t\r\n\f]+') +_match_number = _compile(r'[+-]?(?:[0-9]*\.[0-9]+|[0-9]+)') +_match_hash = _compile('#(?:%(nmchar)s)+') +_match_ident = _compile('-?(?:%(nmstart)s)(?:%(nmchar)s)*') +_match_string_by_quote = { + "'": _compile(r"([^\n\r\f\\']|%(string_escape)s)*"), + '"': _compile(r'([^\n\r\f\\"]|%(string_escape)s)*'), +} + +_sub_simple_escape = re.compile(r'\\(.)').sub +_sub_unicode_escape = re.compile(TokenMacros.unicode_escape, re.I).sub +_sub_newline_escape =re.compile(r'\\(?:\n|\r\n|\r|\f)').sub + +# Same as r'\1', but faster on CPython +_replace_simple = operator.methodcaller('group', 1) + +def _replace_unicode(match): + codepoint = int(match.group(1), 16) + if codepoint > sys.maxunicode: + codepoint = 0xFFFD + return _unichr(codepoint) + + +def unescape_ident(value): + value = _sub_unicode_escape(_replace_unicode, value) + value = _sub_simple_escape(_replace_simple, value) + return value + + +def tokenize(s): + pos = 0 + len_s = len(s) + while pos < len_s: + match = _match_whitespace(s, pos=pos) + if match: + yield Token('S', ' ', pos) + pos = match.end() + continue + + match = _match_ident(s, pos=pos) + if match: + value = _sub_simple_escape(_replace_simple, + _sub_unicode_escape(_replace_unicode, match.group())) + yield Token('IDENT', value, pos) + pos = match.end() + continue + + match = _match_hash(s, pos=pos) + if match: + value = _sub_simple_escape(_replace_simple, + _sub_unicode_escape(_replace_unicode, match.group()[1:])) + yield Token('HASH', value, pos) + pos = match.end() + continue + + quote = s[pos] + if quote in _match_string_by_quote: + match = _match_string_by_quote[quote](s, pos=pos + 1) + assert match, 'Should have found at least an empty match' + end_pos = match.end() + if end_pos == len_s: + raise SelectorSyntaxError('Unclosed string at %s' % pos) + if s[end_pos] != quote: + raise SelectorSyntaxError('Invalid string at %s' % pos) + value = _sub_simple_escape(_replace_simple, + _sub_unicode_escape(_replace_unicode, + _sub_newline_escape('', match.group()))) + yield Token('STRING', value, pos) + pos = end_pos + 1 + continue + + match = _match_number(s, pos=pos) + if match: + value = match.group() + yield Token('NUMBER', value, pos) + pos = match.end() + continue + + pos2 = pos + 2 + if s[pos:pos2] == '/*': + pos = s.find('*/', pos2) + if pos == -1: + pos = len_s + else: + pos += 2 + continue + + yield Token('DELIM', s[pos], pos) + pos += 1 + + assert pos == len_s + yield EOFToken(pos) + + +class TokenStream(object): + def __init__(self, tokens, source=None): + self.used = [] + self.tokens = iter(tokens) + self.source = source + self.peeked = None + self._peeking = False + try: + self.next_token = self.tokens.next + except AttributeError: + # Python 3 + self.next_token = self.tokens.__next__ + + def next(self): + if self._peeking: + self._peeking = False + self.used.append(self.peeked) + return self.peeked + else: + next = self.next_token() + self.used.append(next) + return next + + def peek(self): + if not self._peeking: + self.peeked = self.next_token() + self._peeking = True + return self.peeked + + def next_ident(self): + next = self.next() + if next.type != 'IDENT': + raise SelectorSyntaxError('Expected ident, got %s' % (next,)) + return next.value + + def next_ident_or_star(self): + next = self.next() + if next.type == 'IDENT': + return next.value + elif next == ('DELIM', '*'): + return None + else: + raise SelectorSyntaxError( + "Expected ident or '*', got %s" % (next,)) + + def skip_whitespace(self): + peek = self.peek() + if peek.type == 'S': + self.next() diff --git a/WebCrawler/venv/Lib/site-packages/cssselect/xpath.py b/WebCrawler/venv/Lib/site-packages/cssselect/xpath.py new file mode 100644 index 0000000..db50c77 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/cssselect/xpath.py @@ -0,0 +1,783 @@ +# -*- coding: utf-8 -*- +""" + cssselect.xpath + =============== + + Translation of parsed CSS selectors to XPath expressions. + + + :copyright: (c) 2007-2012 Ian Bicking and contributors. + See AUTHORS for more details. + :license: BSD, see LICENSE for more details. + +""" + +import sys +import re + +from cssselect.parser import parse, parse_series, SelectorError + + +if sys.version_info[0] < 3: + _basestring = basestring + _unicode = unicode +else: + _basestring = str + _unicode = str + + +def _unicode_safe_getattr(obj, name, default=None): + # getattr() with a non-ASCII name fails on Python 2.x + name = name.encode('ascii', 'replace').decode('ascii') + return getattr(obj, name, default) + + +class ExpressionError(SelectorError, RuntimeError): + """Unknown or unsupported selector (eg. pseudo-class).""" + + +#### XPath Helpers + +class XPathExpr(object): + + def __init__(self, path='', element='*', condition='', star_prefix=False): + self.path = path + self.element = element + self.condition = condition + + def __str__(self): + path = _unicode(self.path) + _unicode(self.element) + if self.condition: + path += '[%s]' % self.condition + return path + + def __repr__(self): + return '%s[%s]' % (self.__class__.__name__, self) + + def add_condition(self, condition): + if self.condition: + self.condition = '%s and (%s)' % (self.condition, condition) + else: + self.condition = condition + return self + + def add_name_test(self): + if self.element == '*': + # We weren't doing a test anyway + return + self.add_condition( + "name() = %s" % GenericTranslator.xpath_literal(self.element)) + self.element = '*' + + def add_star_prefix(self): + """ + Append '*/' to the path to keep the context constrained + to a single parent. + """ + self.path += '*/' + + def join(self, combiner, other): + path = _unicode(self) + combiner + # Any "star prefix" is redundant when joining. + if other.path != '*/': + path += other.path + self.path = path + self.element = other.element + self.condition = other.condition + return self + + +split_at_single_quotes = re.compile("('+)").split + +# The spec is actually more permissive than that, but don’t bother. +# This is just for the fast path. +# http://www.w3.org/TR/REC-xml/#NT-NameStartChar +is_safe_name = re.compile('^[a-zA-Z_][a-zA-Z0-9_.-]*$').match + +# Test that the string is not empty and does not contain whitespace +is_non_whitespace = re.compile(r'^[^ \t\r\n\f]+$').match + + +#### Translation + +class GenericTranslator(object): + """ + Translator for "generic" XML documents. + + Everything is case-sensitive, no assumption is made on the meaning + of element names and attribute names. + + """ + + #### + #### HERE BE DRAGONS + #### + #### You are welcome to hook into this to change some behavior, + #### but do so at your own risks. + #### Until it has received a lot more work and review, + #### I reserve the right to change this API in backward-incompatible ways + #### with any minor version of cssselect. + #### See https://github.com/scrapy/cssselect/pull/22 + #### -- Simon Sapin. + #### + + combinator_mapping = { + ' ': 'descendant', + '>': 'child', + '+': 'direct_adjacent', + '~': 'indirect_adjacent', + } + + attribute_operator_mapping = { + 'exists': 'exists', + '=': 'equals', + '~=': 'includes', + '|=': 'dashmatch', + '^=': 'prefixmatch', + '$=': 'suffixmatch', + '*=': 'substringmatch', + '!=': 'different', # XXX Not in Level 3 but meh + } + + #: The attribute used for ID selectors depends on the document language: + #: http://www.w3.org/TR/selectors/#id-selectors + id_attribute = 'id' + + #: The attribute used for ``:lang()`` depends on the document language: + #: http://www.w3.org/TR/selectors/#lang-pseudo + lang_attribute = 'xml:lang' + + #: The case sensitivity of document language element names, + #: attribute names, and attribute values in selectors depends + #: on the document language. + #: http://www.w3.org/TR/selectors/#casesens + #: + #: When a document language defines one of these as case-insensitive, + #: cssselect assumes that the document parser makes the parsed values + #: lower-case. Making the selector lower-case too makes the comparaison + #: case-insensitive. + #: + #: In HTML, element names and attributes names (but not attribute values) + #: are case-insensitive. All of lxml.html, html5lib, BeautifulSoup4 + #: and HTMLParser make them lower-case in their parse result, so + #: the assumption holds. + lower_case_element_names = False + lower_case_attribute_names = False + lower_case_attribute_values = False + + # class used to represent and xpath expression + xpathexpr_cls = XPathExpr + + def css_to_xpath(self, css, prefix='descendant-or-self::'): + """Translate a *group of selectors* to XPath. + + Pseudo-elements are not supported here since XPath only knows + about "real" elements. + + :param css: + A *group of selectors* as an Unicode string. + :param prefix: + This string is prepended to the XPath expression for each selector. + The default makes selectors scoped to the context node’s subtree. + :raises: + :class:`SelectorSyntaxError` on invalid selectors, + :class:`ExpressionError` on unknown/unsupported selectors, + including pseudo-elements. + :returns: + The equivalent XPath 1.0 expression as an Unicode string. + + """ + return ' | '.join(self.selector_to_xpath(selector, prefix, + translate_pseudo_elements=True) + for selector in parse(css)) + + def selector_to_xpath(self, selector, prefix='descendant-or-self::', + translate_pseudo_elements=False): + """Translate a parsed selector to XPath. + + + :param selector: + A parsed :class:`Selector` object. + :param prefix: + This string is prepended to the resulting XPath expression. + The default makes selectors scoped to the context node’s subtree. + :param translate_pseudo_elements: + Unless this is set to ``True`` (as :meth:`css_to_xpath` does), + the :attr:`~Selector.pseudo_element` attribute of the selector + is ignored. + It is the caller's responsibility to reject selectors + with pseudo-elements, or to account for them somehow. + :raises: + :class:`ExpressionError` on unknown/unsupported selectors. + :returns: + The equivalent XPath 1.0 expression as an Unicode string. + + """ + tree = getattr(selector, 'parsed_tree', None) + if not tree: + raise TypeError('Expected a parsed selector, got %r' % (selector,)) + xpath = self.xpath(tree) + assert isinstance(xpath, self.xpathexpr_cls) # help debug a missing 'return' + if translate_pseudo_elements and selector.pseudo_element: + xpath = self.xpath_pseudo_element(xpath, selector.pseudo_element) + return (prefix or '') + _unicode(xpath) + + def xpath_pseudo_element(self, xpath, pseudo_element): + """Translate a pseudo-element. + + Defaults to not supporting pseudo-elements at all, + but can be overridden by sub-classes. + + """ + raise ExpressionError('Pseudo-elements are not supported.') + + @staticmethod + def xpath_literal(s): + s = _unicode(s) + if "'" not in s: + s = "'%s'" % s + elif '"' not in s: + s = '"%s"' % s + else: + s = "concat(%s)" % ','.join([ + (("'" in part) and '"%s"' or "'%s'") % part + for part in split_at_single_quotes(s) if part + ]) + return s + + def xpath(self, parsed_selector): + """Translate any parsed selector object.""" + type_name = type(parsed_selector).__name__ + method = getattr(self, 'xpath_%s' % type_name.lower(), None) + if method is None: + raise ExpressionError('%s is not supported.' % type_name) + return method(parsed_selector) + + + # Dispatched by parsed object type + + def xpath_combinedselector(self, combined): + """Translate a combined selector.""" + combinator = self.combinator_mapping[combined.combinator] + method = getattr(self, 'xpath_%s_combinator' % combinator) + return method(self.xpath(combined.selector), + self.xpath(combined.subselector)) + + def xpath_negation(self, negation): + xpath = self.xpath(negation.selector) + sub_xpath = self.xpath(negation.subselector) + sub_xpath.add_name_test() + if sub_xpath.condition: + return xpath.add_condition('not(%s)' % sub_xpath.condition) + else: + return xpath.add_condition('0') + + def xpath_function(self, function): + """Translate a functional pseudo-class.""" + method = 'xpath_%s_function' % function.name.replace('-', '_') + method = _unicode_safe_getattr(self, method, None) + if not method: + raise ExpressionError( + "The pseudo-class :%s() is unknown" % function.name) + return method(self.xpath(function.selector), function) + + def xpath_pseudo(self, pseudo): + """Translate a pseudo-class.""" + method = 'xpath_%s_pseudo' % pseudo.ident.replace('-', '_') + method = _unicode_safe_getattr(self, method, None) + if not method: + # TODO: better error message for pseudo-elements? + raise ExpressionError( + "The pseudo-class :%s is unknown" % pseudo.ident) + return method(self.xpath(pseudo.selector)) + + + def xpath_attrib(self, selector): + """Translate an attribute selector.""" + operator = self.attribute_operator_mapping[selector.operator] + method = getattr(self, 'xpath_attrib_%s' % operator) + if self.lower_case_attribute_names: + name = selector.attrib.lower() + else: + name = selector.attrib + safe = is_safe_name(name) + if selector.namespace: + name = '%s:%s' % (selector.namespace, name) + safe = safe and is_safe_name(selector.namespace) + if safe: + attrib = '@' + name + else: + attrib = 'attribute::*[name() = %s]' % self.xpath_literal(name) + if selector.value is None: + value = None + elif self.lower_case_attribute_values: + value = selector.value.value.lower() + else: + value = selector.value.value + return method(self.xpath(selector.selector), attrib, value) + + def xpath_class(self, class_selector): + """Translate a class selector.""" + # .foo is defined as [class~=foo] in the spec. + xpath = self.xpath(class_selector.selector) + return self.xpath_attrib_includes( + xpath, '@class', class_selector.class_name) + + def xpath_hash(self, id_selector): + """Translate an ID selector.""" + xpath = self.xpath(id_selector.selector) + return self.xpath_attrib_equals(xpath, '@id', id_selector.id) + + def xpath_element(self, selector): + """Translate a type or universal selector.""" + element = selector.element + if not element: + element = '*' + safe = True + else: + safe = is_safe_name(element) + if self.lower_case_element_names: + element = element.lower() + if selector.namespace: + # Namespace prefixes are case-sensitive. + # http://www.w3.org/TR/css3-namespace/#prefixes + element = '%s:%s' % (selector.namespace, element) + safe = safe and is_safe_name(selector.namespace) + xpath = self.xpathexpr_cls(element=element) + if not safe: + xpath.add_name_test() + return xpath + + + # CombinedSelector: dispatch by combinator + + def xpath_descendant_combinator(self, left, right): + """right is a child, grand-child or further descendant of left""" + return left.join('/descendant-or-self::*/', right) + + def xpath_child_combinator(self, left, right): + """right is an immediate child of left""" + return left.join('/', right) + + def xpath_direct_adjacent_combinator(self, left, right): + """right is a sibling immediately after left""" + xpath = left.join('/following-sibling::', right) + xpath.add_name_test() + return xpath.add_condition('position() = 1') + + def xpath_indirect_adjacent_combinator(self, left, right): + """right is a sibling after left, immediately or not""" + return left.join('/following-sibling::', right) + + + # Function: dispatch by function/pseudo-class name + + def xpath_nth_child_function(self, xpath, function, last=False, + add_name_test=True): + try: + a, b = parse_series(function.arguments) + except ValueError: + raise ExpressionError("Invalid series: '%r'" % function.arguments) + + # From https://www.w3.org/TR/css3-selectors/#structural-pseudos: + # + # :nth-child(an+b) + # an+b-1 siblings before + # + # :nth-last-child(an+b) + # an+b-1 siblings after + # + # :nth-of-type(an+b) + # an+b-1 siblings with the same expanded element name before + # + # :nth-last-of-type(an+b) + # an+b-1 siblings with the same expanded element name after + # + # So, + # for :nth-child and :nth-of-type + # + # count(preceding-sibling::) = an+b-1 + # + # for :nth-last-child and :nth-last-of-type + # + # count(following-sibling::) = an+b-1 + # + # therefore, + # count(...) - (b-1) ≡ 0 (mod a) + # + # if a == 0: + # ~~~~~~~~~~ + # count(...) = b-1 + # + # if a < 0: + # ~~~~~~~~~ + # count(...) - b +1 <= 0 + # -> count(...) <= b-1 + # + # if a > 0: + # ~~~~~~~~~ + # count(...) - b +1 >= 0 + # -> count(...) >= b-1 + + # work with b-1 instead + b_min_1 = b - 1 + + # early-exit condition 1: + # ~~~~~~~~~~~~~~~~~~~~~~~ + # for a == 1, nth-*(an+b) means n+b-1 siblings before/after, + # and since n ∈ {0, 1, 2, ...}, if b-1<=0, + # there is always an "n" matching any number of siblings (maybe none) + if a == 1 and b_min_1 <=0: + return xpath + + # early-exit condition 2: + # ~~~~~~~~~~~~~~~~~~~~~~~ + # an+b-1 siblings with a<0 and (b-1)<0 is not possible + if a < 0 and b_min_1 < 0: + return xpath.add_condition('0') + + # `add_name_test` boolean is inverted and somewhat counter-intuitive: + # + # nth_of_type() calls nth_child(add_name_test=False) + if add_name_test: + nodetest = '*' + else: + nodetest = '%s' % xpath.element + + # count siblings before or after the element + if not last: + siblings_count = 'count(preceding-sibling::%s)' % nodetest + else: + siblings_count = 'count(following-sibling::%s)' % nodetest + + # special case of fixed position: nth-*(0n+b) + # if a == 0: + # ~~~~~~~~~~ + # count(***-sibling::***) = b-1 + if a == 0: + return xpath.add_condition('%s = %s' % (siblings_count, b_min_1)) + + expr = [] + + if a > 0: + # siblings count, an+b-1, is always >= 0, + # so if a>0, and (b-1)<=0, an "n" exists to satisfy this, + # therefore, the predicate is only interesting if (b-1)>0 + if b_min_1 > 0: + expr.append('%s >= %s' % (siblings_count, b_min_1)) + else: + # if a<0, and (b-1)<0, no "n" satisfies this, + # this is tested above as an early exist condition + # otherwise, + expr.append('%s <= %s' % (siblings_count, b_min_1)) + + # operations modulo 1 or -1 are simpler, one only needs to verify: + # + # - either: + # count(***-sibling::***) - (b-1) = n = 0, 1, 2, 3, etc., + # i.e. count(***-sibling::***) >= (b-1) + # + # - or: + # count(***-sibling::***) - (b-1) = -n = 0, -1, -2, -3, etc., + # i.e. count(***-sibling::***) <= (b-1) + # we we just did above. + # + if abs(a) != 1: + # count(***-sibling::***) - (b-1) ≡ 0 (mod a) + left = siblings_count + + # apply "modulo a" on 2nd term, -(b-1), + # to simplify things like "(... +6) % -3", + # and also make it positive with |a| + b_neg = (-b_min_1) % abs(a) + + if b_neg != 0: + b_neg = '+%s' % b_neg + left = '(%s %s)' % (left, b_neg) + + expr.append('%s mod %s = 0' % (left, a)) + + xpath.add_condition(' and '.join(expr)) + return xpath + + def xpath_nth_last_child_function(self, xpath, function): + return self.xpath_nth_child_function(xpath, function, last=True) + + def xpath_nth_of_type_function(self, xpath, function): + if xpath.element == '*': + raise ExpressionError( + "*:nth-of-type() is not implemented") + return self.xpath_nth_child_function(xpath, function, + add_name_test=False) + + def xpath_nth_last_of_type_function(self, xpath, function): + if xpath.element == '*': + raise ExpressionError( + "*:nth-of-type() is not implemented") + return self.xpath_nth_child_function(xpath, function, last=True, + add_name_test=False) + + def xpath_contains_function(self, xpath, function): + # Defined there, removed in later drafts: + # http://www.w3.org/TR/2001/CR-css3-selectors-20011113/#content-selectors + if function.argument_types() not in (['STRING'], ['IDENT']): + raise ExpressionError( + "Expected a single string or ident for :contains(), got %r" + % function.arguments) + value = function.arguments[0].value + return xpath.add_condition( + 'contains(., %s)' % self.xpath_literal(value)) + + def xpath_lang_function(self, xpath, function): + if function.argument_types() not in (['STRING'], ['IDENT']): + raise ExpressionError( + "Expected a single string or ident for :lang(), got %r" + % function.arguments) + value = function.arguments[0].value + return xpath.add_condition( + "lang(%s)" % (self.xpath_literal(value))) + + + # Pseudo: dispatch by pseudo-class name + + def xpath_root_pseudo(self, xpath): + return xpath.add_condition("not(parent::*)") + + # CSS immediate children (CSS ":scope > div" to XPath "child::div" or "./div") + # Works only at the start of a selector + # Needed to get immediate children of a processed selector in Scrapy + # for product in response.css('.product'): + # description = product.css(':scope > div::text').get() + def xpath_scope_pseudo(self, xpath): + return xpath.add_condition("1") + + def xpath_first_child_pseudo(self, xpath): + return xpath.add_condition('count(preceding-sibling::*) = 0') + + def xpath_last_child_pseudo(self, xpath): + return xpath.add_condition('count(following-sibling::*) = 0') + + def xpath_first_of_type_pseudo(self, xpath): + if xpath.element == '*': + raise ExpressionError( + "*:first-of-type is not implemented") + return xpath.add_condition('count(preceding-sibling::%s) = 0' % xpath.element) + + def xpath_last_of_type_pseudo(self, xpath): + if xpath.element == '*': + raise ExpressionError( + "*:last-of-type is not implemented") + return xpath.add_condition('count(following-sibling::%s) = 0' % xpath.element) + + def xpath_only_child_pseudo(self, xpath): + return xpath.add_condition('count(parent::*/child::*) = 1') + + def xpath_only_of_type_pseudo(self, xpath): + if xpath.element == '*': + raise ExpressionError( + "*:only-of-type is not implemented") + return xpath.add_condition('count(parent::*/child::%s) = 1' % xpath.element) + + def xpath_empty_pseudo(self, xpath): + return xpath.add_condition("not(*) and not(string-length())") + + def pseudo_never_matches(self, xpath): + """Common implementation for pseudo-classes that never match.""" + return xpath.add_condition("0") + + xpath_link_pseudo = pseudo_never_matches + xpath_visited_pseudo = pseudo_never_matches + xpath_hover_pseudo = pseudo_never_matches + xpath_active_pseudo = pseudo_never_matches + xpath_focus_pseudo = pseudo_never_matches + xpath_target_pseudo = pseudo_never_matches + xpath_enabled_pseudo = pseudo_never_matches + xpath_disabled_pseudo = pseudo_never_matches + xpath_checked_pseudo = pseudo_never_matches + + # Attrib: dispatch by attribute operator + + def xpath_attrib_exists(self, xpath, name, value): + assert not value + xpath.add_condition(name) + return xpath + + def xpath_attrib_equals(self, xpath, name, value): + xpath.add_condition('%s = %s' % (name, self.xpath_literal(value))) + return xpath + + def xpath_attrib_different(self, xpath, name, value): + # FIXME: this seems like a weird hack... + if value: + xpath.add_condition('not(%s) or %s != %s' + % (name, name, self.xpath_literal(value))) + else: + xpath.add_condition('%s != %s' + % (name, self.xpath_literal(value))) + return xpath + + def xpath_attrib_includes(self, xpath, name, value): + if is_non_whitespace(value): + xpath.add_condition( + "%s and contains(concat(' ', normalize-space(%s), ' '), %s)" + % (name, name, self.xpath_literal(' '+value+' '))) + else: + xpath.add_condition('0') + return xpath + + def xpath_attrib_dashmatch(self, xpath, name, value): + # Weird, but true... + xpath.add_condition('%s and (%s = %s or starts-with(%s, %s))' % ( + name, + name, self.xpath_literal(value), + name, self.xpath_literal(value + '-'))) + return xpath + + def xpath_attrib_prefixmatch(self, xpath, name, value): + if value: + xpath.add_condition('%s and starts-with(%s, %s)' % ( + name, name, self.xpath_literal(value))) + else: + xpath.add_condition('0') + return xpath + + def xpath_attrib_suffixmatch(self, xpath, name, value): + if value: + # Oddly there is a starts-with in XPath 1.0, but not ends-with + xpath.add_condition( + '%s and substring(%s, string-length(%s)-%s) = %s' + % (name, name, name, len(value)-1, self.xpath_literal(value))) + else: + xpath.add_condition('0') + return xpath + + def xpath_attrib_substringmatch(self, xpath, name, value): + if value: + # Attribute selectors are case sensitive + xpath.add_condition('%s and contains(%s, %s)' % ( + name, name, self.xpath_literal(value))) + else: + xpath.add_condition('0') + return xpath + + +class HTMLTranslator(GenericTranslator): + """ + Translator for (X)HTML documents. + + Has a more useful implementation of some pseudo-classes based on + HTML-specific element names and attribute names, as described in + the `HTML5 specification`_. It assumes no-quirks mode. + The API is the same as :class:`GenericTranslator`. + + .. _HTML5 specification: http://www.w3.org/TR/html5/links.html#selectors + + :param xhtml: + If false (the default), element names and attribute names + are case-insensitive. + + """ + + lang_attribute = 'lang' + + def __init__(self, xhtml=False): + self.xhtml = xhtml # Might be useful for sub-classes? + if not xhtml: + # See their definition in GenericTranslator. + self.lower_case_element_names = True + self.lower_case_attribute_names = True + + def xpath_checked_pseudo(self, xpath): + # FIXME: is this really all the elements? + return xpath.add_condition( + "(@selected and name(.) = 'option') or " + "(@checked " + "and (name(.) = 'input' or name(.) = 'command')" + "and (@type = 'checkbox' or @type = 'radio'))") + + def xpath_lang_function(self, xpath, function): + if function.argument_types() not in (['STRING'], ['IDENT']): + raise ExpressionError( + "Expected a single string or ident for :lang(), got %r" + % function.arguments) + value = function.arguments[0].value + return xpath.add_condition( + "ancestor-or-self::*[@lang][1][starts-with(concat(" + # XPath 1.0 has no lower-case function... + "translate(@%s, 'ABCDEFGHIJKLMNOPQRSTUVWXYZ', " + "'abcdefghijklmnopqrstuvwxyz'), " + "'-'), %s)]" + % (self.lang_attribute, self.xpath_literal(value.lower() + '-'))) + + def xpath_link_pseudo(self, xpath): + return xpath.add_condition("@href and " + "(name(.) = 'a' or name(.) = 'link' or name(.) = 'area')") + + # Links are never visited, the implementation for :visited is the same + # as in GenericTranslator + + def xpath_disabled_pseudo(self, xpath): + # http://www.w3.org/TR/html5/section-index.html#attributes-1 + return xpath.add_condition(''' + ( + @disabled and + ( + (name(.) = 'input' and @type != 'hidden') or + name(.) = 'button' or + name(.) = 'select' or + name(.) = 'textarea' or + name(.) = 'command' or + name(.) = 'fieldset' or + name(.) = 'optgroup' or + name(.) = 'option' + ) + ) or ( + ( + (name(.) = 'input' and @type != 'hidden') or + name(.) = 'button' or + name(.) = 'select' or + name(.) = 'textarea' + ) + and ancestor::fieldset[@disabled] + ) + ''') + # FIXME: in the second half, add "and is not a descendant of that + # fieldset element's first legend element child, if any." + + def xpath_enabled_pseudo(self, xpath): + # http://www.w3.org/TR/html5/section-index.html#attributes-1 + return xpath.add_condition(''' + ( + @href and ( + name(.) = 'a' or + name(.) = 'link' or + name(.) = 'area' + ) + ) or ( + ( + name(.) = 'command' or + name(.) = 'fieldset' or + name(.) = 'optgroup' + ) + and not(@disabled) + ) or ( + ( + (name(.) = 'input' and @type != 'hidden') or + name(.) = 'button' or + name(.) = 'select' or + name(.) = 'textarea' or + name(.) = 'keygen' + ) + and not (@disabled or ancestor::fieldset[@disabled]) + ) or ( + name(.) = 'option' and not( + @disabled or ancestor::optgroup[@disabled] + ) + ) + ''') + # FIXME: ... or "li elements that are children of menu elements, + # and that have a child element that defines a command, if the first + # such element's Disabled State facet is false (not disabled)". + # FIXME: after ancestor::fieldset[@disabled], add "and is not a + # descendant of that fieldset element's first legend element child, + # if any." diff --git a/WebCrawler/venv/Lib/site-packages/easy_install.py b/WebCrawler/venv/Lib/site-packages/easy_install.py new file mode 100644 index 0000000..d87e984 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/easy_install.py @@ -0,0 +1,5 @@ +"""Run the EasyInstall command""" + +if __name__ == '__main__': + from setuptools.command.easy_install import main + main() diff --git a/WebCrawler/venv/Lib/site-packages/exampleproj/__init__.py b/WebCrawler/venv/Lib/site-packages/exampleproj/__init__.py new file mode 100644 index 0000000..c188749 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/exampleproj/__init__.py @@ -0,0 +1,13 @@ +""" +An example project. + +@added: exampleproj NEXT +""" + +from incremental import Version +from ._version import __version__ + +__all__ = ["__version__"] + +if Version("exampleproj", "NEXT", 0, 0) > __version__: + print("Unreleased!") diff --git a/WebCrawler/venv/Lib/site-packages/exampleproj/__pycache__/__init__.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/exampleproj/__pycache__/__init__.cpython-39.pyc new file mode 100644 index 0000000..8de9e99 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/exampleproj/__pycache__/__init__.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/exampleproj/__pycache__/_version.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/exampleproj/__pycache__/_version.cpython-39.pyc new file mode 100644 index 0000000..f699f0b Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/exampleproj/__pycache__/_version.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/exampleproj/_version.py b/WebCrawler/venv/Lib/site-packages/exampleproj/_version.py new file mode 100644 index 0000000..21dcfdb --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/exampleproj/_version.py @@ -0,0 +1,7 @@ +# This file is auto-generated! Do not edit! +# Use `python -m incremental.update exampleproj` to change this file. + +from incremental import Version +__version__ = Version('exampleproj', 1, 2, 3) + +__all__ = ["__version__"] diff --git a/WebCrawler/venv/Lib/site-packages/hamcrest/__init__.py b/WebCrawler/venv/Lib/site-packages/hamcrest/__init__.py new file mode 100644 index 0000000..7a36257 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/hamcrest/__init__.py @@ -0,0 +1,7 @@ +from hamcrest.core import * +from hamcrest.library import * + +__version__ = "2.0.2" +__author__ = "Chris Rose" +__copyright__ = "Copyright 2020 hamcrest.org" +__license__ = "BSD, see License.txt" diff --git a/WebCrawler/venv/Lib/site-packages/hamcrest/__pycache__/__init__.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/hamcrest/__pycache__/__init__.cpython-39.pyc new file mode 100644 index 0000000..11fb866 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/hamcrest/__pycache__/__init__.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/hamcrest/core/__init__.py b/WebCrawler/venv/Lib/site-packages/hamcrest/core/__init__.py new file mode 100644 index 0000000..d71b14b --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/hamcrest/core/__init__.py @@ -0,0 +1,6 @@ +from hamcrest.core.assert_that import assert_that +from hamcrest.core.core import * + +__author__ = "Jon Reid" +__copyright__ = "Copyright 2011 hamcrest.org" +__license__ = "BSD, see License.txt" diff --git a/WebCrawler/venv/Lib/site-packages/hamcrest/core/__pycache__/__init__.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/hamcrest/core/__pycache__/__init__.cpython-39.pyc new file mode 100644 index 0000000..8ec1b6f Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/hamcrest/core/__pycache__/__init__.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/hamcrest/core/__pycache__/assert_that.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/hamcrest/core/__pycache__/assert_that.cpython-39.pyc new file mode 100644 index 0000000..29fdb7a Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/hamcrest/core/__pycache__/assert_that.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/hamcrest/core/__pycache__/base_description.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/hamcrest/core/__pycache__/base_description.cpython-39.pyc new file mode 100644 index 0000000..9d0d5dd Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/hamcrest/core/__pycache__/base_description.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/hamcrest/core/__pycache__/base_matcher.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/hamcrest/core/__pycache__/base_matcher.cpython-39.pyc new file mode 100644 index 0000000..2c205da Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/hamcrest/core/__pycache__/base_matcher.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/hamcrest/core/__pycache__/compat.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/hamcrest/core/__pycache__/compat.cpython-39.pyc new file mode 100644 index 0000000..d43486a Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/hamcrest/core/__pycache__/compat.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/hamcrest/core/__pycache__/description.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/hamcrest/core/__pycache__/description.cpython-39.pyc new file mode 100644 index 0000000..a4b11d1 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/hamcrest/core/__pycache__/description.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/hamcrest/core/__pycache__/matcher.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/hamcrest/core/__pycache__/matcher.cpython-39.pyc new file mode 100644 index 0000000..80b9071 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/hamcrest/core/__pycache__/matcher.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/hamcrest/core/__pycache__/selfdescribing.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/hamcrest/core/__pycache__/selfdescribing.cpython-39.pyc new file mode 100644 index 0000000..72f77b6 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/hamcrest/core/__pycache__/selfdescribing.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/hamcrest/core/__pycache__/selfdescribingvalue.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/hamcrest/core/__pycache__/selfdescribingvalue.cpython-39.pyc new file mode 100644 index 0000000..1f9010e Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/hamcrest/core/__pycache__/selfdescribingvalue.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/hamcrest/core/__pycache__/string_description.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/hamcrest/core/__pycache__/string_description.cpython-39.pyc new file mode 100644 index 0000000..f6452b1 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/hamcrest/core/__pycache__/string_description.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/hamcrest/core/assert_that.py b/WebCrawler/venv/Lib/site-packages/hamcrest/core/assert_that.py new file mode 100644 index 0000000..c8882bb --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/hamcrest/core/assert_that.py @@ -0,0 +1,80 @@ +import warnings +from typing import Optional, TypeVar, cast, overload + +from hamcrest.core.matcher import Matcher +from hamcrest.core.string_description import StringDescription + +__author__ = "Jon Reid" +__copyright__ = "Copyright 2011 hamcrest.org" +__license__ = "BSD, see License.txt" +# unittest integration; hide these frames from tracebacks +__unittest = True +# py.test integration; hide these frames from tracebacks +__tracebackhide__ = True + +T = TypeVar("T") + + +@overload +def assert_that(actual: T, matcher: Matcher[T], reason: str = "") -> None: + ... + + +@overload +def assert_that(assertion: bool, reason: str = "") -> None: + ... + + +def assert_that(actual, matcher=None, reason=""): + """Asserts that actual value satisfies matcher. (Can also assert plain + boolean condition.) + + :param actual: The object to evaluate as the actual value. + :param matcher: The matcher to satisfy as the expected condition. + :param reason: Optional explanation to include in failure description. + + ``assert_that`` passes the actual value to the matcher for evaluation. If + the matcher is not satisfied, an exception is thrown describing the + mismatch. + + ``assert_that`` is designed to integrate well with PyUnit and other unit + testing frameworks. The exception raised for an unmet assertion is an + :py:exc:`AssertionError`, which PyUnit reports as a test failure. + + With a different set of parameters, ``assert_that`` can also verify a + boolean condition: + + .. function:: assert_that(assertion[, reason]) + + :param assertion: Boolean condition to verify. + :param reason: Optional explanation to include in failure description. + + This is equivalent to the :py:meth:`~unittest.TestCase.assertTrue` method + of :py:class:`unittest.TestCase`, but offers greater flexibility in test + writing by being a standalone function. + + """ + if isinstance(matcher, Matcher): + _assert_match(actual=actual, matcher=matcher, reason=reason) + else: + if isinstance(actual, Matcher): + warnings.warn("arg1 should be boolean, but was {}".format(type(actual))) + _assert_bool(assertion=cast(bool, actual), reason=cast(str, matcher)) + + +def _assert_match(actual: T, matcher: Matcher[T], reason: str) -> None: + if not matcher.matches(actual): + description = StringDescription() + description.append_text(reason).append_text("\nExpected: ").append_description_of( + matcher + ).append_text("\n but: ") + matcher.describe_mismatch(actual, description) + description.append_text("\n") + raise AssertionError(description) + + +def _assert_bool(assertion: bool, reason: Optional[str] = None) -> None: + if not assertion: + if not reason: + reason = "Assertion failed" + raise AssertionError(reason) diff --git a/WebCrawler/venv/Lib/site-packages/hamcrest/core/base_description.py b/WebCrawler/venv/Lib/site-packages/hamcrest/core/base_description.py new file mode 100644 index 0000000..e0ceea3 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/hamcrest/core/base_description.py @@ -0,0 +1,70 @@ +from typing import Any, Iterable + +from hamcrest.core.description import Description +from hamcrest.core.helpers.hasmethod import hasmethod +from hamcrest.core.helpers.ismock import ismock + +__author__ = "Jon Reid" +__copyright__ = "Copyright 2011 hamcrest.org" +__license__ = "BSD, see License.txt" + + +class BaseDescription(Description): + """Base class for all :py:class:`~hamcrest.core.description.Description` + implementations. + + """ + + def append_text(self, text: str) -> Description: + self.append(text) + return self + + def append_description_of(self, value: Any) -> Description: + if not ismock(value) and hasmethod(value, "describe_to"): + value.describe_to(self) + elif isinstance(value, str): + self.append(repr(value)) + else: + description = str(value) + if description[:1] == "<" and description[-1:] == ">": + self.append(description) + else: + self.append("<") + self.append(description) + self.append(">") + return self + + def append_list(self, start: str, separator: str, end: str, list: Iterable[Any]) -> Description: + separate = False + + self.append(start) + for item in list: + if separate: + self.append(separator) + self.append_description_of(item) + separate = True + self.append(end) + return self + + def append(self, string: str) -> None: + """Append the string to the description.""" + raise NotImplementedError("append") + + def append_string_in_python_syntax(self, string: str) -> None: + self.append("'") + for ch in string: + self.append(character_in_python_syntax(ch)) + self.append("'") + + +def character_in_python_syntax(ch: str) -> str: + if ch == "'": + return "'" + elif ch == "\n": + return "\\n" + elif ch == "\r": + return "\\r" + elif ch == "\t": + return "\\t" + else: + return ch diff --git a/WebCrawler/venv/Lib/site-packages/hamcrest/core/base_matcher.py b/WebCrawler/venv/Lib/site-packages/hamcrest/core/base_matcher.py new file mode 100644 index 0000000..c1c01ea --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/hamcrest/core/base_matcher.py @@ -0,0 +1,41 @@ +from typing import Optional, TypeVar + +from hamcrest.core.description import Description +from hamcrest.core.matcher import Matcher +from hamcrest.core.string_description import tostring + +__author__ = "Jon Reid" +__copyright__ = "Copyright 2011 hamcrest.org" +__license__ = "BSD, see License.txt" + +T = TypeVar("T") + + +class BaseMatcher(Matcher[T]): + """Base class for all :py:class:`~hamcrest.core.matcher.Matcher` + implementations. + + Most implementations can just implement :py:obj:`_matches`, leaving the + handling of any mismatch description to the ``matches`` method. But if it + makes more sense to generate the mismatch description during the matching, + override :py:meth:`~hamcrest.core.matcher.Matcher.matches` instead. + + """ + + def __str__(self) -> str: + return tostring(self) + + def _matches(self, item: T) -> bool: + raise NotImplementedError("_matches") + + def matches(self, item: T, mismatch_description: Optional[Description] = None) -> bool: + match_result = self._matches(item) + if not match_result and mismatch_description: + self.describe_mismatch(item, mismatch_description) + return match_result + + def describe_mismatch(self, item: T, mismatch_description: Description) -> None: + mismatch_description.append_text("was ").append_description_of(item) + + def describe_match(self, item: T, match_description: Description) -> None: + match_description.append_text("was ").append_description_of(item) diff --git a/WebCrawler/venv/Lib/site-packages/hamcrest/core/compat.py b/WebCrawler/venv/Lib/site-packages/hamcrest/core/compat.py new file mode 100644 index 0000000..2c6d1fa --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/hamcrest/core/compat.py @@ -0,0 +1,19 @@ +__author__ = "Per Fagrell" +__copyright__ = "Copyright 2013 hamcrest.org" +__license__ = "BSD, see License.txt" + +__all__ = ['is_callable'] + +import sys + +# callable was not part of py3k until 3.2, so we create this +# generic is_callable to use callable if possible, otherwise +# we use generic homebrew. +if sys.version_info[0] == 3 and sys.version_info[1] < 2: + def is_callable(function): + """Return whether the object is callable (i.e., some kind of function).""" + if function is None: + return False + return any("__call__" in klass.__dict__ for klass in type(function).__mro__) +else: + is_callable = callable diff --git a/WebCrawler/venv/Lib/site-packages/hamcrest/core/core/__init__.py b/WebCrawler/venv/Lib/site-packages/hamcrest/core/core/__init__.py new file mode 100644 index 0000000..6f452c1 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/hamcrest/core/core/__init__.py @@ -0,0 +1,17 @@ +"""Fundamental matchers of objects and values, and composite matchers.""" + +from hamcrest.core.core.allof import all_of +from hamcrest.core.core.anyof import any_of +from hamcrest.core.core.described_as import described_as +from hamcrest.core.core.is_ import is_ +from hamcrest.core.core.isanything import anything +from hamcrest.core.core.isequal import equal_to +from hamcrest.core.core.isinstanceof import instance_of +from hamcrest.core.core.isnone import none, not_none +from hamcrest.core.core.isnot import is_not, not_ +from hamcrest.core.core.issame import same_instance +from hamcrest.core.core.raises import calling, raises + +__author__ = "Jon Reid" +__copyright__ = "Copyright 2011 hamcrest.org" +__license__ = "BSD, see License.txt" diff --git a/WebCrawler/venv/Lib/site-packages/hamcrest/core/core/__pycache__/__init__.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/hamcrest/core/core/__pycache__/__init__.cpython-39.pyc new file mode 100644 index 0000000..63238ee Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/hamcrest/core/core/__pycache__/__init__.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/hamcrest/core/core/__pycache__/allof.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/hamcrest/core/core/__pycache__/allof.cpython-39.pyc new file mode 100644 index 0000000..aaa8045 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/hamcrest/core/core/__pycache__/allof.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/hamcrest/core/core/__pycache__/anyof.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/hamcrest/core/core/__pycache__/anyof.cpython-39.pyc new file mode 100644 index 0000000..bf6ccf4 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/hamcrest/core/core/__pycache__/anyof.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/hamcrest/core/core/__pycache__/described_as.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/hamcrest/core/core/__pycache__/described_as.cpython-39.pyc new file mode 100644 index 0000000..2b7d460 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/hamcrest/core/core/__pycache__/described_as.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/hamcrest/core/core/__pycache__/is_.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/hamcrest/core/core/__pycache__/is_.cpython-39.pyc new file mode 100644 index 0000000..85c5dbd Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/hamcrest/core/core/__pycache__/is_.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/hamcrest/core/core/__pycache__/isanything.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/hamcrest/core/core/__pycache__/isanything.cpython-39.pyc new file mode 100644 index 0000000..fb35e21 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/hamcrest/core/core/__pycache__/isanything.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/hamcrest/core/core/__pycache__/isequal.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/hamcrest/core/core/__pycache__/isequal.cpython-39.pyc new file mode 100644 index 0000000..2c2a75a Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/hamcrest/core/core/__pycache__/isequal.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/hamcrest/core/core/__pycache__/isinstanceof.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/hamcrest/core/core/__pycache__/isinstanceof.cpython-39.pyc new file mode 100644 index 0000000..10f2c23 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/hamcrest/core/core/__pycache__/isinstanceof.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/hamcrest/core/core/__pycache__/isnone.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/hamcrest/core/core/__pycache__/isnone.cpython-39.pyc new file mode 100644 index 0000000..3646927 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/hamcrest/core/core/__pycache__/isnone.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/hamcrest/core/core/__pycache__/isnot.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/hamcrest/core/core/__pycache__/isnot.cpython-39.pyc new file mode 100644 index 0000000..1b26c84 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/hamcrest/core/core/__pycache__/isnot.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/hamcrest/core/core/__pycache__/issame.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/hamcrest/core/core/__pycache__/issame.cpython-39.pyc new file mode 100644 index 0000000..835b437 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/hamcrest/core/core/__pycache__/issame.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/hamcrest/core/core/__pycache__/raises.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/hamcrest/core/core/__pycache__/raises.cpython-39.pyc new file mode 100644 index 0000000..6543cde Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/hamcrest/core/core/__pycache__/raises.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/hamcrest/core/core/allof.py b/WebCrawler/venv/Lib/site-packages/hamcrest/core/core/allof.py new file mode 100644 index 0000000..ffb62f6 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/hamcrest/core/core/allof.py @@ -0,0 +1,58 @@ +from typing import Optional, TypeVar, Union + +from hamcrest.core.base_matcher import BaseMatcher +from hamcrest.core.description import Description +from hamcrest.core.helpers.wrap_matcher import wrap_matcher +from hamcrest.core.matcher import Matcher + +__author__ = "Jon Reid" +__copyright__ = "Copyright 2011 hamcrest.org" +__license__ = "BSD, see License.txt" + +T = TypeVar("T") + + +class AllOf(BaseMatcher[T]): + def __init__(self, *matchers: Matcher[T], **kwargs): + self.matchers = matchers + self.describe_matcher_in_mismatch = kwargs.pop( + "describe_matcher_in_mismatch", True + ) # No keyword-only args in 2.7 :-( + self.describe_all_mismatches = kwargs.pop("describe_all_mismatches", False) + + def matches(self, item: T, mismatch_description: Optional[Description] = None) -> bool: + found_mismatch = False + for i, matcher in enumerate(self.matchers): + if not matcher.matches(item): + if mismatch_description: + if self.describe_matcher_in_mismatch: + mismatch_description.append_description_of(matcher).append_text(" ") + matcher.describe_mismatch(item, mismatch_description) + found_mismatch = True + if not self.describe_all_mismatches: + break + elif i < len(self.matchers) - 1 and mismatch_description: + mismatch_description.append_text(" and ") + return not found_mismatch + + def describe_mismatch(self, item: T, mismatch_description: Description) -> None: + self.matches(item, mismatch_description) + + def describe_to(self, description: Description) -> None: + description.append_list("(", " and ", ")", self.matchers) + + +def all_of(*items: Union[Matcher[T], T]) -> Matcher[T]: + """Matches if all of the given matchers evaluate to ``True``. + + :param matcher1,...: A comma-separated list of matchers. + + The matchers are evaluated from left to right using short-circuit + evaluation, so evaluation stops as soon as a matcher returns ``False``. + + Any argument that is not a matcher is implicitly wrapped in an + :py:func:`~hamcrest.core.core.isequal.equal_to` matcher to check for + equality. + + """ + return AllOf(*[wrap_matcher(item) for item in items]) diff --git a/WebCrawler/venv/Lib/site-packages/hamcrest/core/core/anyof.py b/WebCrawler/venv/Lib/site-packages/hamcrest/core/core/anyof.py new file mode 100644 index 0000000..921e3b2 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/hamcrest/core/core/anyof.py @@ -0,0 +1,42 @@ +from typing import TypeVar, Union + +from hamcrest.core.base_matcher import BaseMatcher +from hamcrest.core.description import Description +from hamcrest.core.helpers.wrap_matcher import wrap_matcher +from hamcrest.core.matcher import Matcher + +__author__ = "Jon Reid" +__copyright__ = "Copyright 2011 hamcrest.org" +__license__ = "BSD, see License.txt" + +T = TypeVar("T") + + +class AnyOf(BaseMatcher[T]): + def __init__(self, *matchers: Matcher[T]) -> None: + self.matchers = matchers + + def _matches(self, item: T) -> bool: + for matcher in self.matchers: + if matcher.matches(item): + return True + return False + + def describe_to(self, description: Description) -> None: + description.append_list("(", " or ", ")", self.matchers) + + +def any_of(*items: Union[Matcher[T], T]) -> Matcher[T]: + """Matches if any of the given matchers evaluate to ``True``. + + :param matcher1,...: A comma-separated list of matchers. + + The matchers are evaluated from left to right using short-circuit + evaluation, so evaluation stops as soon as a matcher returns ``True``. + + Any argument that is not a matcher is implicitly wrapped in an + :py:func:`~hamcrest.core.core.isequal.equal_to` matcher to check for + equality. + + """ + return AnyOf(*[wrap_matcher(item) for item in items]) diff --git a/WebCrawler/venv/Lib/site-packages/hamcrest/core/core/described_as.py b/WebCrawler/venv/Lib/site-packages/hamcrest/core/core/described_as.py new file mode 100644 index 0000000..dba2b5c --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/hamcrest/core/core/described_as.py @@ -0,0 +1,52 @@ +import re +from typing import Any, Optional, Tuple + +from hamcrest.core.base_matcher import BaseMatcher +from hamcrest.core.description import Description +from hamcrest.core.matcher import Matcher + +__author__ = "Jon Reid" +__copyright__ = "Copyright 2011 hamcrest.org" +__license__ = "BSD, see License.txt" + +ARG_PATTERN = re.compile("%([0-9]+)") + + +class DescribedAs(BaseMatcher[Any]): + def __init__( + self, description_template: str, matcher: Matcher[Any], *values: Tuple[Any, ...] + ) -> None: + self.template = description_template + self.matcher = matcher + self.values = values + + def matches(self, item: Any, mismatch_description: Optional[Description] = None) -> bool: + return self.matcher.matches(item, mismatch_description) + + def describe_mismatch(self, item: Any, mismatch_description: Description) -> None: + self.matcher.describe_mismatch(item, mismatch_description) + + def describe_to(self, description: Description) -> None: + text_start = 0 + for match in re.finditer(ARG_PATTERN, self.template): + description.append_text(self.template[text_start : match.start()]) + arg_index = int(match.group()[1:]) + description.append_description_of(self.values[arg_index]) + text_start = match.end() + + if text_start < len(self.template): + description.append_text(self.template[text_start:]) + + +def described_as(description: str, matcher: Matcher[Any], *values) -> Matcher[Any]: + """Adds custom failure description to a given matcher. + + :param description: Overrides the matcher's description. + :param matcher: The matcher to satisfy. + :param value1,...: Optional comma-separated list of substitution values. + + The description may contain substitution placeholders %0, %1, etc. These + will be replaced by any values that follow the matcher. + + """ + return DescribedAs(description, matcher, *values) diff --git a/WebCrawler/venv/Lib/site-packages/hamcrest/core/core/is_.py b/WebCrawler/venv/Lib/site-packages/hamcrest/core/core/is_.py new file mode 100644 index 0000000..6b1b541 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/hamcrest/core/core/is_.py @@ -0,0 +1,101 @@ +from typing import Optional, Type, TypeVar, Union, overload + +from hamcrest.core.base_matcher import BaseMatcher +from hamcrest.core.description import Description +from hamcrest.core.helpers.wrap_matcher import is_matchable_type, wrap_matcher +from hamcrest.core.matcher import Matcher + +from .isinstanceof import instance_of + +__author__ = "Jon Reid" +__copyright__ = "Copyright 2011 hamcrest.org" +__license__ = "BSD, see License.txt" + +T = TypeVar("T") + + +class Is(BaseMatcher[T]): + def __init__(self, matcher: Matcher[T]) -> None: + self.matcher = matcher + + def matches(self, item: T, mismatch_description: Optional[Description] = None) -> bool: + return self.matcher.matches(item, mismatch_description) + + def describe_mismatch(self, item: T, mismatch_description: Description) -> None: + return self.matcher.describe_mismatch(item, mismatch_description) + + def describe_to(self, description: Description): + description.append_description_of(self.matcher) + + +@overload +def _wrap_value_or_type(x: Type) -> Matcher[object]: + ... + + +@overload +def _wrap_value_or_type(x: T) -> Matcher[T]: + ... + + +def _wrap_value_or_type(x): + if is_matchable_type(x): + return instance_of(x) + else: + return wrap_matcher(x) + + +@overload +def is_(x: Type) -> Matcher[object]: + ... + + +@overload +def is_(x: Union[Matcher[T], T]) -> Matcher[T]: + ... + + +def is_(x): + """Decorates another matcher, or provides shortcuts to the frequently used + ``is(equal_to(x))`` and ``is(instance_of(x))``. + + :param x: The matcher to satisfy, or a type for + :py:func:`~hamcrest.core.core.isinstanceof.instance_of` matching, or an + expected value for :py:func:`~hamcrest.core.core.isequal.equal_to` + matching. + + This matcher compares the evaluated object to the given matcher. + + .. note:: + + PyHamcrest's ``is_`` matcher is unrelated to Python's ``is`` operator. + The matcher for object identity is + :py:func:`~hamcrest.core.core.issame.same_instance`. + + If the ``x`` argument is a matcher, its behavior is retained, but the test + may be more expressive. For example:: + + assert_that(value, less_than(5)) + assert_that(value, is_(less_than(5))) + + If the ``x`` argument is a type, it is wrapped in an + :py:func:`~hamcrest.core.core.isinstanceof.instance_of` matcher. This makes + the following statements equivalent:: + + assert_that(cheese, instance_of(Cheddar)) + assert_that(cheese, is_(instance_of(Cheddar))) + assert_that(cheese, is_(Cheddar)) + + Otherwise, if the ``x`` argument is not a matcher, it is wrapped in an + :py:func:`~hamcrest.core.core.isequal.equal_to` matcher. This makes the + following statements equivalent:: + + assert_that(cheese, equal_to(smelly)) + assert_that(cheese, is_(equal_to(smelly))) + assert_that(cheese, is_(smelly)) + + Choose the style that makes your expression most readable. This will vary + depending on context. + + """ + return Is(_wrap_value_or_type(x)) diff --git a/WebCrawler/venv/Lib/site-packages/hamcrest/core/core/isanything.py b/WebCrawler/venv/Lib/site-packages/hamcrest/core/core/isanything.py new file mode 100644 index 0000000..5fa4550 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/hamcrest/core/core/isanything.py @@ -0,0 +1,32 @@ +from typing import Any, Optional + +from hamcrest.core.base_matcher import BaseMatcher +from hamcrest.core.description import Description +from hamcrest.core.matcher import Matcher + +__author__ = "Jon Reid" +__copyright__ = "Copyright 2011 hamcrest.org" +__license__ = "BSD, see License.txt" + + +class IsAnything(BaseMatcher[Any]): + def __init__(self, description: Optional[str]) -> None: + self.description = description or "ANYTHING" # type: str + + def _matches(self, item: Any) -> bool: + return True + + def describe_to(self, description: Description) -> None: + description.append_text(self.description) + + +def anything(description: Optional[str] = None) -> Matcher[Any]: + """Matches anything. + + :param description: Optional string used to describe this matcher. + + This matcher always evaluates to ``True``. Specify this in composite + matchers when the value of a particular element is unimportant. + + """ + return IsAnything(description) diff --git a/WebCrawler/venv/Lib/site-packages/hamcrest/core/core/isequal.py b/WebCrawler/venv/Lib/site-packages/hamcrest/core/core/isequal.py new file mode 100644 index 0000000..1816091 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/hamcrest/core/core/isequal.py @@ -0,0 +1,34 @@ +from typing import Any + +from hamcrest.core.base_matcher import BaseMatcher +from hamcrest.core.description import Description +from hamcrest.core.matcher import Matcher + +__author__ = "Jon Reid" +__copyright__ = "Copyright 2011 hamcrest.org" +__license__ = "BSD, see License.txt" + + +class IsEqual(BaseMatcher[Any]): + def __init__(self, equals: Any) -> None: + self.object = equals + + def _matches(self, item: Any) -> bool: + return item == self.object + + def describe_to(self, description: Description) -> None: + nested_matcher = isinstance(self.object, Matcher) + if nested_matcher: + description.append_text("<") + description.append_description_of(self.object) + if nested_matcher: + description.append_text(">") + + +def equal_to(obj: Any) -> Matcher[Any]: + """Matches if object is equal to a given object. + + :param obj: The object to compare against as the expected value. + + This matcher compares the evaluated object to ``obj`` for equality.""" + return IsEqual(obj) diff --git a/WebCrawler/venv/Lib/site-packages/hamcrest/core/core/isinstanceof.py b/WebCrawler/venv/Lib/site-packages/hamcrest/core/core/isinstanceof.py new file mode 100644 index 0000000..a34aaa6 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/hamcrest/core/core/isinstanceof.py @@ -0,0 +1,39 @@ +from typing import Type + +from hamcrest.core.base_matcher import BaseMatcher +from hamcrest.core.description import Description +from hamcrest.core.helpers.wrap_matcher import is_matchable_type +from hamcrest.core.matcher import Matcher + +__author__ = "Jon Reid" +__copyright__ = "Copyright 2011 hamcrest.org" +__license__ = "BSD, see License.txt" + + +class IsInstanceOf(BaseMatcher[object]): + def __init__(self, expected_type: Type) -> None: + if not is_matchable_type(expected_type): + raise TypeError("IsInstanceOf requires type") + self.expected_type = expected_type + + def _matches(self, item: object) -> bool: + return isinstance(item, self.expected_type) + + def describe_to(self, description: Description) -> None: + description.append_text("an instance of ").append_text(self.expected_type.__name__) + + +def instance_of(atype: Type) -> Matcher[object]: + """Matches if object is an instance of, or inherits from, a given type. + + :param atype: The type to compare against as the expected type. + + This matcher checks whether the evaluated object is an instance of + ``atype`` or an instance of any class that inherits from ``atype``. + + Example:: + + instance_of(str) + + """ + return IsInstanceOf(atype) diff --git a/WebCrawler/venv/Lib/site-packages/hamcrest/core/core/isnone.py b/WebCrawler/venv/Lib/site-packages/hamcrest/core/core/isnone.py new file mode 100644 index 0000000..1b4fbc1 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/hamcrest/core/core/isnone.py @@ -0,0 +1,29 @@ +from typing import Any, Optional + +from hamcrest.core.base_matcher import BaseMatcher +from hamcrest.core.description import Description +from hamcrest.core.matcher import Matcher + +from .isnot import is_not + +__author__ = "Jon Reid" +__copyright__ = "Copyright 2011 hamcrest.org" +__license__ = "BSD, see License.txt" + + +class IsNone(BaseMatcher[Optional[Any]]): + def _matches(self, item: Any) -> bool: + return item is None + + def describe_to(self, description: Description) -> None: + description.append_text("None") + + +def none() -> Matcher[Optional[Any]]: + """Matches if object is ``None``.""" + return IsNone() + + +def not_none() -> Matcher[Optional[Any]]: + """Matches if object is not ``None``.""" + return is_not(none()) diff --git a/WebCrawler/venv/Lib/site-packages/hamcrest/core/core/isnot.py b/WebCrawler/venv/Lib/site-packages/hamcrest/core/core/isnot.py new file mode 100644 index 0000000..de26f68 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/hamcrest/core/core/isnot.py @@ -0,0 +1,86 @@ +from typing import Type, TypeVar, Union, overload + +from hamcrest.core.base_matcher import BaseMatcher +from hamcrest.core.description import Description +from hamcrest.core.helpers.wrap_matcher import is_matchable_type, wrap_matcher +from hamcrest.core.matcher import Matcher + +from .isinstanceof import instance_of + +__author__ = "Jon Reid" +__copyright__ = "Copyright 2011 hamcrest.org" +__license__ = "BSD, see License.txt" + +T = TypeVar("T") + + +class IsNot(BaseMatcher[T]): + def __init__(self, matcher: Matcher[T]) -> None: + self.matcher = matcher + + def _matches(self, item: T) -> bool: + return not self.matcher.matches(item) + + def describe_to(self, description: Description) -> None: + description.append_text("not ").append_description_of(self.matcher) + + def describe_mismatch(self, item: T, mismatch_description: Description) -> None: + mismatch_description.append_text("but ") + self.matcher.describe_match(item, mismatch_description) + + +@overload +def _wrap_value_or_type(x: Type) -> Matcher[object]: + ... + + +@overload +def _wrap_value_or_type(x: T) -> Matcher[T]: + ... + + +def _wrap_value_or_type(x): + if is_matchable_type(x): + return instance_of(x) + else: + return wrap_matcher(x) + + +@overload +def is_not(match: Type) -> Matcher[object]: + ... + + +@overload +def is_not(match: Union[Matcher[T], T]) -> Matcher[T]: + ... + + +def is_not(match): + """Inverts the given matcher to its logical negation. + + :param match: The matcher to negate. + + This matcher compares the evaluated object to the negation of the given + matcher. If the ``match`` argument is not a matcher, it is implicitly + wrapped in an :py:func:`~hamcrest.core.core.isequal.equal_to` matcher to + check for equality, and thus matches for inequality. + + Examples:: + + assert_that(cheese, is_not(equal_to(smelly))) + assert_that(cheese, is_not(smelly)) + + """ + return IsNot(_wrap_value_or_type(match)) + + +def not_(match: Union[Matcher[T], T]) -> Matcher[T]: + """Alias of :py:func:`is_not` for better readability of negations. + + Examples:: + + assert_that(alist, not_(has_item(item))) + + """ + return is_not(match) diff --git a/WebCrawler/venv/Lib/site-packages/hamcrest/core/core/issame.py b/WebCrawler/venv/Lib/site-packages/hamcrest/core/core/issame.py new file mode 100644 index 0000000..d426944 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/hamcrest/core/core/issame.py @@ -0,0 +1,42 @@ +from typing import TypeVar + +from hamcrest.core.base_matcher import BaseMatcher +from hamcrest.core.description import Description +from hamcrest.core.matcher import Matcher + +__author__ = "Jon Reid" +__copyright__ = "Copyright 2011 hamcrest.org" +__license__ = "BSD, see License.txt" + +T = TypeVar("T") + + +class IsSame(BaseMatcher[T]): + def __init__(self, obj: T) -> None: + self.object = obj + + def _matches(self, item: T) -> bool: + return item is self.object + + def describe_to(self, description: Description) -> None: + description.append_text("same instance as ").append_text(hex(id(self.object))).append_text( + " " + ).append_description_of(self.object) + + def describe_mismatch(self, item: T, mismatch_description: Description) -> None: + mismatch_description.append_text("was ") + if item is not None: + mismatch_description.append_text(hex(id(item))).append_text(" ") + mismatch_description.append_description_of(item) + + +def same_instance(obj: T) -> Matcher[T]: + """Matches if evaluated object is the same instance as a given object. + + :param obj: The object to compare against as the expected value. + + This matcher invokes the ``is`` identity operator to determine if the + evaluated object is the the same object as ``obj``. + + """ + return IsSame(obj) diff --git a/WebCrawler/venv/Lib/site-packages/hamcrest/core/core/raises.py b/WebCrawler/venv/Lib/site-packages/hamcrest/core/core/raises.py new file mode 100644 index 0000000..afb7fd9 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/hamcrest/core/core/raises.py @@ -0,0 +1,141 @@ +import re +import sys +from typing import Any, Callable, Mapping, Optional, Tuple, Type, cast +from weakref import ref + +from hamcrest.core.base_matcher import BaseMatcher +from hamcrest.core.description import Description +from hamcrest.core.matcher import Matcher + +__author__ = "Per Fagrell" +__copyright__ = "Copyright 2013 hamcrest.org" +__license__ = "BSD, see License.txt" + + +class Raises(BaseMatcher[Callable[..., Any]]): + def __init__( + self, + expected: Type[Exception], + pattern: Optional[str] = None, + matching: Optional[Matcher] = None, + ) -> None: + self.pattern = pattern + self.matcher = matching + self.expected = expected + self.actual = None # type: Optional[BaseException] + self.function = None # type: Optional[Callable[..., Any]] + + def _matches(self, function: Callable[..., Any]) -> bool: + if not callable(function): + return False + + self.function = cast(Callable[..., Any], ref(function)) + return self._call_function(function) + + def _call_function(self, function: Callable[..., Any]) -> bool: + self.actual = None + try: + function() + except BaseException: + self.actual = sys.exc_info()[1] + + if isinstance(self.actual, self.expected): + if self.pattern is not None: + if re.search(self.pattern, str(self.actual)) is None: + return False + if self.matcher is not None: + if not self.matcher.matches(self.actual): + return False + return True + return False + + def describe_to(self, description: Description) -> None: + description.append_text("Expected a callable raising %s" % self.expected) + + def describe_mismatch(self, item, description: Description) -> None: + if not callable(item): + description.append_text("%s is not callable" % item) + return + + function = None if self.function is None else self.function() + if function is None or function is not item: + self.function = ref(item) + if not self._call_function(item): + return + + if self.actual is None: + description.append_text("No exception raised.") + elif isinstance(self.actual, self.expected): + if self.pattern is not None or self.matcher is not None: + description.append_text("Correct assertion type raised, but ") + if self.pattern is not None: + description.append_text('the expected pattern ("%s") ' % self.pattern) + if self.pattern is not None and self.matcher is not None: + description.append_text("and ") + if self.matcher is not None: + description.append_description_of(self.matcher) + description.append_text(" ") + description.append_text('not found. Exception message was: "%s"' % str(self.actual)) + else: + description.append_text( + "%r of type %s was raised instead" % (self.actual, type(self.actual)) + ) + + def describe_match(self, item, match_description: Description) -> None: + self._call_function(item) + match_description.append_text( + "%r of type %s was raised." % (self.actual, type(self.actual)) + ) + + +def raises(exception: Type[Exception], pattern=None, matching=None) -> Matcher[Callable[..., Any]]: + """Matches if the called function raised the expected exception. + + :param exception: The class of the expected exception + :param pattern: Optional regular expression to match exception message. + :param matching: Optional Hamcrest matchers to apply to the exception. + + Expects the actual to be wrapped by using :py:func:`~hamcrest.core.core.raises.calling`, + or a callable taking no arguments. + Optional argument pattern should be a string containing a regular expression. If provided, + the string representation of the actual exception - e.g. `str(actual)` - must match pattern. + + Examples:: + + assert_that(calling(int).with_args('q'), raises(TypeError)) + assert_that(calling(parse, broken_input), raises(ValueError)) + assert_that( + calling(valid_user, bad_json), + raises(HTTPError, matching=has_properties(status_code=500) + ) + """ + return Raises(exception, pattern, matching) + + +class DeferredCallable(object): + def __init__(self, func: Callable[..., Any]): + self.func = func + self.args = tuple() # type: Tuple[Any, ...] + self.kwargs = {} # type: Mapping[str, Any] + + def __call__(self): + self.func(*self.args, **self.kwargs) + + def with_args(self, *args, **kwargs): + self.args = args + self.kwargs = kwargs + return self + + +def calling(func: Callable[..., Any]) -> DeferredCallable: + """Wrapper for function call that delays the actual execution so that + :py:func:`~hamcrest.core.core.raises.raises` matcher can catch any thrown exception. + + :param func: The function or method to be called + + The arguments can be provided with a call to the `with_args` function on the returned + object:: + + calling(my_method).with_args(arguments, and_='keywords') + """ + return DeferredCallable(func) diff --git a/WebCrawler/venv/Lib/site-packages/hamcrest/core/description.py b/WebCrawler/venv/Lib/site-packages/hamcrest/core/description.py new file mode 100644 index 0000000..2d36ba4 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/hamcrest/core/description.py @@ -0,0 +1,50 @@ +from typing import Any, Iterable, Sequence + +__author__ = "Jon Reid" +__copyright__ = "Copyright 2011 hamcrest.org" +__license__ = "BSD, see License.txt" + + +class Description: + """A description of a :py:class:`~hamcrest.core.matcher.Matcher`. + + A :py:class:`~hamcrest.core.matcher.Matcher` will describe itself to a + description which can later be used for reporting. + + """ + + def append_text(self, text: str) -> "Description": + """Appends some plain text to the description. + + :returns: ``self``, for chaining + + """ + raise NotImplementedError("append_text") + + def append_description_of(self, value: Any) -> "Description": + """Appends description of given value to this description. + + If the value implements + :py:meth:`~hamcrest.core.selfdescribing.SelfDescribing.describe_to`, + then it will be used. + + :returns: ``self``, for chaining + + """ + raise NotImplementedError("append_description_of") + + def append_list( + self, start: str, separator: str, end: str, list: Iterable[Any] + ) -> "Description": + """Appends a list of objects to the description. + + :param start: String that will begin the list description. + :param separator: String that will separate each object in the + description. + :param end: String that will end the list description. + :param list: List of objects to be described. + + :returns: ``self``, for chaining + + """ + raise NotImplementedError("append_list") diff --git a/WebCrawler/venv/Lib/site-packages/hamcrest/core/helpers/__init__.py b/WebCrawler/venv/Lib/site-packages/hamcrest/core/helpers/__init__.py new file mode 100644 index 0000000..61cb82d --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/hamcrest/core/helpers/__init__.py @@ -0,0 +1,5 @@ +"""Utilities for writing Matchers.""" + +__author__ = "Jon Reid" +__copyright__ = "Copyright 2011 hamcrest.org" +__license__ = "BSD, see License.txt" diff --git a/WebCrawler/venv/Lib/site-packages/hamcrest/core/helpers/__pycache__/__init__.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/hamcrest/core/helpers/__pycache__/__init__.cpython-39.pyc new file mode 100644 index 0000000..9e20162 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/hamcrest/core/helpers/__pycache__/__init__.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/hamcrest/core/helpers/__pycache__/hasmethod.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/hamcrest/core/helpers/__pycache__/hasmethod.cpython-39.pyc new file mode 100644 index 0000000..dd60806 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/hamcrest/core/helpers/__pycache__/hasmethod.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/hamcrest/core/helpers/__pycache__/ismock.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/hamcrest/core/helpers/__pycache__/ismock.cpython-39.pyc new file mode 100644 index 0000000..6514b16 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/hamcrest/core/helpers/__pycache__/ismock.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/hamcrest/core/helpers/__pycache__/wrap_matcher.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/hamcrest/core/helpers/__pycache__/wrap_matcher.cpython-39.pyc new file mode 100644 index 0000000..3d94dc9 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/hamcrest/core/helpers/__pycache__/wrap_matcher.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/hamcrest/core/helpers/hasmethod.py b/WebCrawler/venv/Lib/site-packages/hamcrest/core/helpers/hasmethod.py new file mode 100644 index 0000000..fcf4120 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/hamcrest/core/helpers/hasmethod.py @@ -0,0 +1,12 @@ +__author__ = "Jon Reid" +__copyright__ = "Copyright 2011 hamcrest.org" +__license__ = "BSD, see License.txt" + + +def hasmethod(obj: object, methodname: str) -> bool: + """Does ``obj`` have a method named ``methodname``?""" + + if not hasattr(obj, methodname): + return False + method = getattr(obj, methodname) + return callable(method) diff --git a/WebCrawler/venv/Lib/site-packages/hamcrest/core/helpers/ismock.py b/WebCrawler/venv/Lib/site-packages/hamcrest/core/helpers/ismock.py new file mode 100644 index 0000000..26b561b --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/hamcrest/core/helpers/ismock.py @@ -0,0 +1,19 @@ +from typing import Any, List, Type + +MOCKTYPES = [] # type: List[Type] +try: + from mock import Mock + + MOCKTYPES += [Mock] +except ImportError: + pass +try: + from unittest.mock import Mock + + MOCKTYPES += [Mock] +except ImportError: + pass + + +def ismock(obj: Any) -> bool: + return isinstance(obj, tuple(MOCKTYPES)) diff --git a/WebCrawler/venv/Lib/site-packages/hamcrest/core/helpers/wrap_matcher.py b/WebCrawler/venv/Lib/site-packages/hamcrest/core/helpers/wrap_matcher.py new file mode 100644 index 0000000..99d3fc0 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/hamcrest/core/helpers/wrap_matcher.py @@ -0,0 +1,30 @@ +from typing import Type, TypeVar, Union + +from hamcrest.core.base_matcher import Matcher +from hamcrest.core.core.isequal import equal_to + +__author__ = "Jon Reid" +__copyright__ = "Copyright 2011 hamcrest.org" +__license__ = "BSD, see License.txt" + +T = TypeVar("T") + + +def wrap_matcher(x: Union[Matcher[T], T]) -> Matcher[T]: + """Wraps argument in a matcher, if necessary. + + :returns: the argument as-is if it is already a matcher, otherwise wrapped + in an :py:func:`~hamcrest.core.core.isequal.equal_to` matcher. + + """ + if isinstance(x, Matcher): + return x + else: + return equal_to(x) + + +def is_matchable_type(expected_type: Type) -> bool: + if isinstance(expected_type, type): + return True + + return False diff --git a/WebCrawler/venv/Lib/site-packages/hamcrest/core/matcher.py b/WebCrawler/venv/Lib/site-packages/hamcrest/core/matcher.py new file mode 100644 index 0000000..7ee5ece --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/hamcrest/core/matcher.py @@ -0,0 +1,74 @@ +from typing import Generic, Optional, TypeVar + +from hamcrest.core.description import Description + +from .selfdescribing import SelfDescribing + +__author__ = "Jon Reid" +__copyright__ = "Copyright 2011 hamcrest.org" +__license__ = "BSD, see License.txt" + +T = TypeVar("T") + + +class Matcher(Generic[T], SelfDescribing): + """A matcher over acceptable values. + + A matcher is able to describe itself to give feedback when it fails. + + Matcher implementations should *not* directly implement this protocol. + Instead, *extend* the :py:class:`~hamcrest.core.base_matcher.BaseMatcher` + class, which will ensure that the + :py:class:`~hamcrest.core.matcher.Matcher` API can grow to support new + features and remain compatible with all + :py:class:`~hamcrest.core.matcher.Matcher` implementations. + + """ + + def matches(self, item: T, mismatch_description: Optional[Description] = None) -> bool: + """Evaluates the matcher for argument item. + + If a mismatch is detected and argument ``mismatch_description`` is + provided, it will generate a description of why the matcher has not + accepted the item. + + :param item: The object against which the matcher is evaluated. + :param mismatch_description: + :returns: ``True`` if ``item`` matches, otherwise ``False``. + + """ + raise NotImplementedError("matches") + + def describe_mismatch(self, item: T, mismatch_description: Description) -> None: + """Generates a description of why the matcher has not accepted the + item. + + The description will be part of a larger description of why a matching + failed, so it should be concise. + + This method assumes that ``matches(item)`` is ``False``, but will not + check this. + + :param item: The item that the + :py:class:`~hamcrest.core.matcher.Matcher` has rejected. + :param mismatch_description: The description to be built or appended + to. + + """ + raise NotImplementedError("describe_mismatch") + + def describe_match(self, item: T, match_description: Description) -> None: + """Generates a description of why the matcher has accepted the item. + + The description may be part of a larger description of why a matching + failed, so it should be concise. + + This method assumes that ``matches(item)`` is ``True``, but will not + check this. + + :param item: The item that the + :py:class:`~hamcrest.core.matcher.Matcher` has accepted. + :param match_description: The description to be built or appended to. + + """ + raise NotImplementedError("describe_match") diff --git a/WebCrawler/venv/Lib/site-packages/hamcrest/core/selfdescribing.py b/WebCrawler/venv/Lib/site-packages/hamcrest/core/selfdescribing.py new file mode 100644 index 0000000..d57cb6e --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/hamcrest/core/selfdescribing.py @@ -0,0 +1,20 @@ +from hamcrest.core.description import Description + +__author__ = "Jon Reid" +__copyright__ = "Copyright 2011 hamcrest.org" +__license__ = "BSD, see License.txt" + + +class SelfDescribing: + """The ability of an object to describe itself.""" + + def describe_to(self, description: Description) -> None: + """Generates a description of the object. + + The description may be part of a description of a larger object of + which this is just a component, so it should be worded appropriately. + + :param description: The description to be built or appended to. + + """ + raise NotImplementedError("describe_to") diff --git a/WebCrawler/venv/Lib/site-packages/hamcrest/core/selfdescribingvalue.py b/WebCrawler/venv/Lib/site-packages/hamcrest/core/selfdescribingvalue.py new file mode 100644 index 0000000..148ca01 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/hamcrest/core/selfdescribingvalue.py @@ -0,0 +1,28 @@ +import warnings +from typing import Any + +from hamcrest.core.description import Description +from hamcrest.core.selfdescribing import SelfDescribing + +__author__ = "Jon Reid" +__copyright__ = "Copyright 2011 hamcrest.org" +__license__ = "BSD, see License.txt" + + +class SelfDescribingValue(SelfDescribing): + """Wrap any value in a ``SelfDescribingValue`` to satisfy the + :py:class:`~hamcrest.core.selfdescribing.SelfDescribing` interface. + + **Deprecated:** No need for this class now that + :py:meth:`~hamcrest.core.description.Description.append_description_of` + handles any type of value. + + """ + + def __init__(self, value: Any) -> None: + warnings.warn("SelfDescribingValue no longer needed", DeprecationWarning) + self.value = value + + def describe_to(self, description: Description) -> None: + """Generates a description of the value.""" + description.append_description_of(self.value) diff --git a/WebCrawler/venv/Lib/site-packages/hamcrest/core/string_description.py b/WebCrawler/venv/Lib/site-packages/hamcrest/core/string_description.py new file mode 100644 index 0000000..96fbb26 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/hamcrest/core/string_description.py @@ -0,0 +1,35 @@ +from hamcrest.core.selfdescribing import SelfDescribing + +from .base_description import BaseDescription + +__author__ = "Jon Reid" +__copyright__ = "Copyright 2011 hamcrest.org" +__license__ = "BSD, see License.txt" + + +def tostring(selfdescribing: SelfDescribing) -> str: + """Returns the description of a + :py:class:`~hamcrest.core.selfdescribing.SelfDescribing` object as a + string. + + :param selfdescribing: The object to be described. + :returns: The description of the object. + """ + return str(StringDescription().append_description_of(selfdescribing)) + + +class StringDescription(BaseDescription): + """A :py:class:`~hamcrest.core.description.Description` that is stored as a + string. + + """ + + def __init__(self) -> None: + self.out = "" + + def __str__(self) -> str: + """Returns the description.""" + return self.out + + def append(self, string: str) -> None: + self.out += str(string) diff --git a/WebCrawler/venv/Lib/site-packages/hamcrest/library/__init__.py b/WebCrawler/venv/Lib/site-packages/hamcrest/library/__init__.py new file mode 100644 index 0000000..fb02c2b --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/hamcrest/library/__init__.py @@ -0,0 +1,44 @@ +"""Library of Matcher implementations.""" + +from hamcrest.core import * +from hamcrest.library.collection import * +from hamcrest.library.integration import * +from hamcrest.library.number import * +from hamcrest.library.object import * +from hamcrest.library.text import * + +__author__ = "Jon Reid" +__copyright__ = "Copyright 2011 hamcrest.org" +__license__ = "BSD, see License.txt" + +__all__ = [ + "has_entry", + "has_entries", + "has_key", + "has_value", + "is_in", + "empty", + "has_item", + "has_items", + "contains_inanyorder", + "contains", + "contains_exactly", + "only_contains", + "match_equality", + "matches_regexp", + "close_to", + "greater_than", + "greater_than_or_equal_to", + "less_than", + "less_than_or_equal_to", + "has_length", + "has_property", + "has_properties", + "has_string", + "equal_to_ignoring_case", + "equal_to_ignoring_whitespace", + "contains_string", + "ends_with", + "starts_with", + "string_contains_in_order", +] diff --git a/WebCrawler/venv/Lib/site-packages/hamcrest/library/__pycache__/__init__.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/hamcrest/library/__pycache__/__init__.cpython-39.pyc new file mode 100644 index 0000000..53fd7b7 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/hamcrest/library/__pycache__/__init__.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/hamcrest/library/collection/__init__.py b/WebCrawler/venv/Lib/site-packages/hamcrest/library/collection/__init__.py new file mode 100644 index 0000000..d06b818 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/hamcrest/library/collection/__init__.py @@ -0,0 +1,15 @@ +"""Matchers of collections.""" +from .is_empty import empty +from .isdict_containing import has_entry +from .isdict_containingentries import has_entries +from .isdict_containingkey import has_key +from .isdict_containingvalue import has_value +from .isin import is_in +from .issequence_containing import has_item, has_items +from .issequence_containinginanyorder import contains_inanyorder +from .issequence_containinginorder import contains, contains_exactly +from .issequence_onlycontaining import only_contains + +__author__ = "Chris Rose" +__copyright__ = "Copyright 2013 hamcrest.org" +__license__ = "BSD, see License.txt" diff --git a/WebCrawler/venv/Lib/site-packages/hamcrest/library/collection/__pycache__/__init__.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/hamcrest/library/collection/__pycache__/__init__.cpython-39.pyc new file mode 100644 index 0000000..235a153 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/hamcrest/library/collection/__pycache__/__init__.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/hamcrest/library/collection/__pycache__/is_empty.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/hamcrest/library/collection/__pycache__/is_empty.cpython-39.pyc new file mode 100644 index 0000000..7a7b9d0 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/hamcrest/library/collection/__pycache__/is_empty.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/hamcrest/library/collection/__pycache__/isdict_containing.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/hamcrest/library/collection/__pycache__/isdict_containing.cpython-39.pyc new file mode 100644 index 0000000..4ef7c4b Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/hamcrest/library/collection/__pycache__/isdict_containing.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/hamcrest/library/collection/__pycache__/isdict_containingentries.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/hamcrest/library/collection/__pycache__/isdict_containingentries.cpython-39.pyc new file mode 100644 index 0000000..b25463a Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/hamcrest/library/collection/__pycache__/isdict_containingentries.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/hamcrest/library/collection/__pycache__/isdict_containingkey.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/hamcrest/library/collection/__pycache__/isdict_containingkey.cpython-39.pyc new file mode 100644 index 0000000..5823d42 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/hamcrest/library/collection/__pycache__/isdict_containingkey.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/hamcrest/library/collection/__pycache__/isdict_containingvalue.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/hamcrest/library/collection/__pycache__/isdict_containingvalue.cpython-39.pyc new file mode 100644 index 0000000..59eb260 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/hamcrest/library/collection/__pycache__/isdict_containingvalue.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/hamcrest/library/collection/__pycache__/isin.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/hamcrest/library/collection/__pycache__/isin.cpython-39.pyc new file mode 100644 index 0000000..f0a16a9 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/hamcrest/library/collection/__pycache__/isin.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/hamcrest/library/collection/__pycache__/issequence_containing.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/hamcrest/library/collection/__pycache__/issequence_containing.cpython-39.pyc new file mode 100644 index 0000000..d85357e Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/hamcrest/library/collection/__pycache__/issequence_containing.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/hamcrest/library/collection/__pycache__/issequence_containinginanyorder.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/hamcrest/library/collection/__pycache__/issequence_containinginanyorder.cpython-39.pyc new file mode 100644 index 0000000..565ee15 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/hamcrest/library/collection/__pycache__/issequence_containinginanyorder.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/hamcrest/library/collection/__pycache__/issequence_containinginorder.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/hamcrest/library/collection/__pycache__/issequence_containinginorder.cpython-39.pyc new file mode 100644 index 0000000..0159c46 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/hamcrest/library/collection/__pycache__/issequence_containinginorder.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/hamcrest/library/collection/__pycache__/issequence_onlycontaining.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/hamcrest/library/collection/__pycache__/issequence_onlycontaining.cpython-39.pyc new file mode 100644 index 0000000..2a5db0d Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/hamcrest/library/collection/__pycache__/issequence_onlycontaining.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/hamcrest/library/collection/is_empty.py b/WebCrawler/venv/Lib/site-packages/hamcrest/library/collection/is_empty.py new file mode 100644 index 0000000..cff60d6 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/hamcrest/library/collection/is_empty.py @@ -0,0 +1,36 @@ +from typing import Optional, Sized + +from hamcrest.core.base_matcher import BaseMatcher +from hamcrest.core.description import Description +from hamcrest.core.matcher import Matcher + +__author__ = "Chris Rose" +__copyright__ = "Copyright 2012 hamcrest.org" +__license__ = "BSD, see License.txt" + + +class IsEmpty(BaseMatcher[Sized]): + def matches(self, item: Sized, mismatch_description: Optional[Description] = None) -> bool: + try: + if len(item) == 0: + return True + + if mismatch_description: + mismatch_description.append_text("has %d item(s)" % len(item)) + + except TypeError: + if mismatch_description: + mismatch_description.append_text("does not support length") + + return False + + def describe_to(self, description: Description) -> None: + description.append_text("an empty collection") + + +def empty() -> Matcher[Sized]: + """ + This matcher matches any collection-like object that responds to the + __len__ method, and has a length of 0. + """ + return IsEmpty() diff --git a/WebCrawler/venv/Lib/site-packages/hamcrest/library/collection/isdict_containing.py b/WebCrawler/venv/Lib/site-packages/hamcrest/library/collection/isdict_containing.py new file mode 100644 index 0000000..b93ef36 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/hamcrest/library/collection/isdict_containing.py @@ -0,0 +1,61 @@ +from typing import Hashable, Mapping, TypeVar, Union + +from hamcrest.core.base_matcher import BaseMatcher +from hamcrest.core.description import Description +from hamcrest.core.helpers.hasmethod import hasmethod +from hamcrest.core.helpers.wrap_matcher import wrap_matcher +from hamcrest.core.matcher import Matcher + +__author__ = "Jon Reid" +__copyright__ = "Copyright 2011 hamcrest.org" +__license__ = "BSD, see License.txt" + + +K = TypeVar("K", bound=Hashable) # TODO - covariant? +V = TypeVar("V") + + +class IsDictContaining(BaseMatcher[Mapping[K, V]]): + def __init__(self, key_matcher: Matcher[K], value_matcher: Matcher[V]) -> None: + self.key_matcher = key_matcher + self.value_matcher = value_matcher + + def _matches(self, item: Mapping[K, V]) -> bool: + if hasmethod(item, "items"): + for key, value in item.items(): + if self.key_matcher.matches(key) and self.value_matcher.matches(value): + return True + return False + + def describe_to(self, description: Description) -> None: + description.append_text("a dictionary containing [").append_description_of( + self.key_matcher + ).append_text(": ").append_description_of(self.value_matcher).append_text("]") + + +def has_entry( + key_match: Union[K, Matcher[K]], value_match: Union[V, Matcher[V]] +) -> Matcher[Mapping[K, V]]: + """Matches if dictionary contains key-value entry satisfying a given pair + of matchers. + + :param key_match: The matcher to satisfy for the key, or an expected value + for :py:func:`~hamcrest.core.core.isequal.equal_to` matching. + :param value_match: The matcher to satisfy for the value, or an expected + value for :py:func:`~hamcrest.core.core.isequal.equal_to` matching. + + This matcher iterates the evaluated dictionary, searching for any key-value + entry that satisfies ``key_match`` and ``value_match``. If a matching entry + is found, ``has_entry`` is satisfied. + + Any argument that is not a matcher is implicitly wrapped in an + :py:func:`~hamcrest.core.core.isequal.equal_to` matcher to check for + equality. + + Examples:: + + has_entry(equal_to('foo'), equal_to(1)) + has_entry('foo', 1) + + """ + return IsDictContaining(wrap_matcher(key_match), wrap_matcher(value_match)) diff --git a/WebCrawler/venv/Lib/site-packages/hamcrest/library/collection/isdict_containingentries.py b/WebCrawler/venv/Lib/site-packages/hamcrest/library/collection/isdict_containingentries.py new file mode 100644 index 0000000..68a28bc --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/hamcrest/library/collection/isdict_containingentries.py @@ -0,0 +1,160 @@ +from typing import Any, Hashable, Mapping, Optional, TypeVar, Union, overload + +from hamcrest.core.base_matcher import BaseMatcher +from hamcrest.core.description import Description +from hamcrest.core.helpers.wrap_matcher import wrap_matcher +from hamcrest.core.matcher import Matcher + +__author__ = "Jon Reid" +__copyright__ = "Copyright 2011 hamcrest.org" +__license__ = "BSD, see License.txt" + +K = TypeVar("K", bound=Hashable) +V = TypeVar("V") + + +class IsDictContainingEntries(BaseMatcher[Mapping[K, V]]): + def __init__(self, value_matchers) -> None: + self.value_matchers = sorted(value_matchers.items()) + + def _not_a_dictionary( + self, item: Mapping[K, V], mismatch_description: Optional[Description] + ) -> bool: + if mismatch_description: + mismatch_description.append_description_of(item).append_text(" is not a mapping object") + return False + + def matches( + self, item: Mapping[K, V], mismatch_description: Optional[Description] = None + ) -> bool: + for key, value_matcher in self.value_matchers: + + try: + if not key in item: + if mismatch_description: + mismatch_description.append_text("no ").append_description_of( + key + ).append_text(" key in ").append_description_of(item) + return False + except TypeError: + return self._not_a_dictionary(item, mismatch_description) + + try: + actual_value = item[key] + except TypeError: + return self._not_a_dictionary(item, mismatch_description) + + if not value_matcher.matches(actual_value): + if mismatch_description: + mismatch_description.append_text("value for ").append_description_of( + key + ).append_text(" ") + value_matcher.describe_mismatch(actual_value, mismatch_description) + return False + + return True + + def describe_mismatch(self, item: Mapping[K, V], mismatch_description: Description) -> None: + self.matches(item, mismatch_description) + + def describe_keyvalue(self, index: int, value: V, description: Description) -> None: + """Describes key-value pair at given index.""" + description.append_description_of(index).append_text(": ").append_description_of(value) + + def describe_to(self, description: Description) -> None: + description.append_text("a dictionary containing {") + first = True + for key, value in self.value_matchers: + if not first: + description.append_text(", ") + self.describe_keyvalue(key, value, description) + first = False + description.append_text("}") + + +# Keyword argument form +@overload +def has_entries(**keys_valuematchers: Union[Matcher[V], V]) -> Matcher[Mapping[str, V]]: + ... + + +# Key to matcher dict form +@overload +def has_entries(keys_valuematchers: Mapping[K, Union[Matcher[V], V]]) -> Matcher[Mapping[K, V]]: + ... + + +# Alternating key/matcher form +@overload +def has_entries(*keys_valuematchers: Any) -> Matcher[Mapping[Any, Any]]: + ... + + +def has_entries(*keys_valuematchers, **kv_args): + """Matches if dictionary contains entries satisfying a dictionary of keys + and corresponding value matchers. + + :param matcher_dict: A dictionary mapping keys to associated value matchers, + or to expected values for + :py:func:`~hamcrest.core.core.isequal.equal_to` matching. + + Note that the keys must be actual keys, not matchers. Any value argument + that is not a matcher is implicitly wrapped in an + :py:func:`~hamcrest.core.core.isequal.equal_to` matcher to check for + equality. + + Examples:: + + has_entries({'foo':equal_to(1), 'bar':equal_to(2)}) + has_entries({'foo':1, 'bar':2}) + + ``has_entries`` also accepts a list of keyword arguments: + + .. function:: has_entries(keyword1=value_matcher1[, keyword2=value_matcher2[, ...]]) + + :param keyword1: A keyword to look up. + :param valueMatcher1: The matcher to satisfy for the value, or an expected + value for :py:func:`~hamcrest.core.core.isequal.equal_to` matching. + + Examples:: + + has_entries(foo=equal_to(1), bar=equal_to(2)) + has_entries(foo=1, bar=2) + + Finally, ``has_entries`` also accepts a list of alternating keys and their + value matchers: + + .. function:: has_entries(key1, value_matcher1[, ...]) + + :param key1: A key (not a matcher) to look up. + :param valueMatcher1: The matcher to satisfy for the value, or an expected + value for :py:func:`~hamcrest.core.core.isequal.equal_to` matching. + + Examples:: + + has_entries('foo', equal_to(1), 'bar', equal_to(2)) + has_entries('foo', 1, 'bar', 2) + + """ + if len(keys_valuematchers) == 1: + try: + base_dict = keys_valuematchers[0].copy() + for key in base_dict: + base_dict[key] = wrap_matcher(base_dict[key]) + except AttributeError: + raise ValueError( + "single-argument calls to has_entries must pass a dict as the argument" + ) + else: + if len(keys_valuematchers) % 2: + raise ValueError("has_entries requires key-value pairs") + base_dict = {} + for index in range(int(len(keys_valuematchers) / 2)): + base_dict[keys_valuematchers[2 * index]] = wrap_matcher( + keys_valuematchers[2 * index + 1] + ) + + for key, value in kv_args.items(): + base_dict[key] = wrap_matcher(value) + + return IsDictContainingEntries(base_dict) diff --git a/WebCrawler/venv/Lib/site-packages/hamcrest/library/collection/isdict_containingkey.py b/WebCrawler/venv/Lib/site-packages/hamcrest/library/collection/isdict_containingkey.py new file mode 100644 index 0000000..90c7cc7 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/hamcrest/library/collection/isdict_containingkey.py @@ -0,0 +1,54 @@ +from typing import Any, Hashable, Mapping, TypeVar, Union + +from hamcrest.core.base_matcher import BaseMatcher +from hamcrest.core.description import Description +from hamcrest.core.helpers.hasmethod import hasmethod +from hamcrest.core.helpers.wrap_matcher import wrap_matcher +from hamcrest.core.matcher import Matcher + +__author__ = "Jon Reid" +__copyright__ = "Copyright 2011 hamcrest.org" +__license__ = "BSD, see License.txt" + +K = TypeVar("K", bound=Hashable) + + +class IsDictContainingKey(BaseMatcher[Mapping[K, Any]]): + def __init__(self, key_matcher: Matcher[K]) -> None: + self.key_matcher = key_matcher + + def _matches(self, item: Mapping[K, Any]) -> bool: + if hasmethod(item, "keys"): + for key in item.keys(): + if self.key_matcher.matches(key): + return True + return False + + def describe_to(self, description: Description) -> None: + description.append_text("a dictionary containing key ").append_description_of( + self.key_matcher + ) + + +def has_key(key_match: Union[K, Matcher[K]]) -> Matcher[Mapping[K, Any]]: + """Matches if dictionary contains an entry whose key satisfies a given + matcher. + + :param key_match: The matcher to satisfy for the key, or an expected value + for :py:func:`~hamcrest.core.core.isequal.equal_to` matching. + + This matcher iterates the evaluated dictionary, searching for any key-value + entry whose key satisfies the given matcher. If a matching entry is found, + ``has_key`` is satisfied. + + Any argument that is not a matcher is implicitly wrapped in an + :py:func:`~hamcrest.core.core.isequal.equal_to` matcher to check for + equality. + + Examples:: + + has_key(equal_to('foo')) + has_key('foo') + + """ + return IsDictContainingKey(wrap_matcher(key_match)) diff --git a/WebCrawler/venv/Lib/site-packages/hamcrest/library/collection/isdict_containingvalue.py b/WebCrawler/venv/Lib/site-packages/hamcrest/library/collection/isdict_containingvalue.py new file mode 100644 index 0000000..ca5b71a --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/hamcrest/library/collection/isdict_containingvalue.py @@ -0,0 +1,54 @@ +from typing import Any, Mapping, TypeVar, Union + +from hamcrest.core.base_matcher import BaseMatcher +from hamcrest.core.description import Description +from hamcrest.core.helpers.hasmethod import hasmethod +from hamcrest.core.helpers.wrap_matcher import wrap_matcher +from hamcrest.core.matcher import Matcher + +__author__ = "Jon Reid" +__copyright__ = "Copyright 2011 hamcrest.org" +__license__ = "BSD, see License.txt" + +V = TypeVar("V") + + +class IsDictContainingValue(BaseMatcher[Mapping[Any, V]]): + def __init__(self, value_matcher: Matcher[V]) -> None: + self.value_matcher = value_matcher + + def _matches(self, item: Mapping[Any, V]) -> bool: + if hasmethod(item, "values"): + for value in item.values(): + if self.value_matcher.matches(value): + return True + return False + + def describe_to(self, description: Description) -> None: + description.append_text("a dictionary containing value ").append_description_of( + self.value_matcher + ) + + +def has_value(value: Union[V, Matcher[V]]) -> Matcher[Mapping[Any, V]]: + """Matches if dictionary contains an entry whose value satisfies a given + matcher. + + :param value_match: The matcher to satisfy for the value, or an expected + value for :py:func:`~hamcrest.core.core.isequal.equal_to` matching. + + This matcher iterates the evaluated dictionary, searching for any key-value + entry whose value satisfies the given matcher. If a matching entry is + found, ``has_value`` is satisfied. + + Any argument that is not a matcher is implicitly wrapped in an + :py:func:`~hamcrest.core.core.isequal.equal_to` matcher to check for + equality. + + Examples:: + + has_value(equal_to('bar')) + has_value('bar') + + """ + return IsDictContainingValue(wrap_matcher(value)) diff --git a/WebCrawler/venv/Lib/site-packages/hamcrest/library/collection/isin.py b/WebCrawler/venv/Lib/site-packages/hamcrest/library/collection/isin.py new file mode 100644 index 0000000..e958ea3 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/hamcrest/library/collection/isin.py @@ -0,0 +1,34 @@ +from typing import Sequence, TypeVar + +from hamcrest.core.base_matcher import BaseMatcher +from hamcrest.core.description import Description +from hamcrest.core.matcher import Matcher + +__author__ = "Jon Reid" +__copyright__ = "Copyright 2011 hamcrest.org" +__license__ = "BSD, see License.txt" + +T = TypeVar("T") + + +class IsIn(BaseMatcher[T]): + def __init__(self, sequence: Sequence[T]) -> None: + self.sequence = sequence + + def _matches(self, item: T) -> bool: + return item in self.sequence + + def describe_to(self, description: Description) -> None: + description.append_text("one of ").append_list("(", ", ", ")", self.sequence) + + +def is_in(sequence: Sequence[T]) -> Matcher[T]: + """Matches if evaluated object is present in a given sequence. + + :param sequence: The sequence to search. + + This matcher invokes the ``in`` membership operator to determine if the + evaluated object is a member of the sequence. + + """ + return IsIn(sequence) diff --git a/WebCrawler/venv/Lib/site-packages/hamcrest/library/collection/issequence_containing.py b/WebCrawler/venv/Lib/site-packages/hamcrest/library/collection/issequence_containing.py new file mode 100644 index 0000000..2d7bf5c --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/hamcrest/library/collection/issequence_containing.py @@ -0,0 +1,93 @@ +from typing import Sequence, TypeVar, Union, cast + +from hamcrest.core.base_matcher import BaseMatcher +from hamcrest.core.core.allof import all_of +from hamcrest.core.description import Description +from hamcrest.core.helpers.wrap_matcher import wrap_matcher +from hamcrest.core.matcher import Matcher + +__author__ = "Jon Reid" +__copyright__ = "Copyright 2011 hamcrest.org" +__license__ = "BSD, see License.txt" + +T = TypeVar("T") + + +class IsSequenceContaining(BaseMatcher[Sequence[T]]): + def __init__(self, element_matcher: Matcher[T]) -> None: + self.element_matcher = element_matcher + + def _matches(self, item: Sequence[T]) -> bool: + try: + for element in item: + if self.element_matcher.matches(element): + return True + except TypeError: # not a sequence + pass + return False + + def describe_to(self, description: Description) -> None: + description.append_text("a sequence containing ").append_description_of( + self.element_matcher + ) + + +# It'd be great to make use of all_of, but we can't be sure we won't +# be seeing a one-time sequence here (like a generator); see issue #20 +# Instead, we wrap it inside a class that will convert the sequence into +# a concrete list and then hand it off to the all_of matcher. +class IsSequenceContainingEvery(BaseMatcher[Sequence[T]]): + def __init__(self, *element_matchers: Matcher[T]) -> None: + delegates = [cast(Matcher[Sequence[T]], has_item(e)) for e in element_matchers] + self.matcher = all_of(*delegates) # type: Matcher[Sequence[T]] + + def _matches(self, item: Sequence[T]) -> bool: + try: + return self.matcher.matches(list(item)) + except TypeError: + return False + + def describe_mismatch(self, item: Sequence[T], mismatch_description: Description) -> None: + self.matcher.describe_mismatch(item, mismatch_description) + + def describe_to(self, description: Description) -> None: + self.matcher.describe_to(description) + + +def has_item(match: Union[Matcher[T], T]) -> Matcher[Sequence[T]]: + """Matches if any element of sequence satisfies a given matcher. + + :param match: The matcher to satisfy, or an expected value for + :py:func:`~hamcrest.core.core.isequal.equal_to` matching. + + This matcher iterates the evaluated sequence, searching for any element + that satisfies a given matcher. If a matching element is found, + ``has_item`` is satisfied. + + If the ``match`` argument is not a matcher, it is implicitly wrapped in an + :py:func:`~hamcrest.core.core.isequal.equal_to` matcher to check for + equality. + + """ + return IsSequenceContaining(wrap_matcher(match)) + + +def has_items(*items: Union[Matcher[T], T]) -> Matcher[Sequence[T]]: + """Matches if all of the given matchers are satisfied by any elements of + the sequence. + + :param match1,...: A comma-separated list of matchers. + + This matcher iterates the given matchers, searching for any elements in the + evaluated sequence that satisfy them. If each matcher is satisfied, then + ``has_items`` is satisfied. + + Any argument that is not a matcher is implicitly wrapped in an + :py:func:`~hamcrest.core.core.isequal.equal_to` matcher to check for + equality. + + """ + matchers = [] + for item in items: + matchers.append(wrap_matcher(item)) + return IsSequenceContainingEvery(*matchers) diff --git a/WebCrawler/venv/Lib/site-packages/hamcrest/library/collection/issequence_containinginanyorder.py b/WebCrawler/venv/Lib/site-packages/hamcrest/library/collection/issequence_containinginanyorder.py new file mode 100644 index 0000000..431709c --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/hamcrest/library/collection/issequence_containinginanyorder.py @@ -0,0 +1,103 @@ +from typing import MutableSequence, Optional, Sequence, TypeVar, Union, cast + +from hamcrest.core.base_matcher import BaseMatcher +from hamcrest.core.description import Description +from hamcrest.core.helpers.wrap_matcher import wrap_matcher +from hamcrest.core.matcher import Matcher + +__author__ = "Jon Reid" +__copyright__ = "Copyright 2011 hamcrest.org" +__license__ = "BSD, see License.txt" + +T = TypeVar("T") + + +class MatchInAnyOrder(object): + def __init__( + self, matchers: Sequence[Matcher[T]], mismatch_description: Optional[Description] + ) -> None: + self.matchers = cast(MutableSequence[Matcher[T]], matchers[:]) + self.mismatch_description = mismatch_description + + def matches(self, item: T) -> bool: + return self.isnotsurplus(item) and self.ismatched(item) + + def isfinished(self, item: Sequence[T]) -> bool: + if not self.matchers: + return True + if self.mismatch_description: + self.mismatch_description.append_text("no item matches: ").append_list( + "", ", ", "", self.matchers + ).append_text(" in ").append_list("[", ", ", "]", item) + return False + + def isnotsurplus(self, item: T) -> bool: + if not self.matchers: + if self.mismatch_description: + self.mismatch_description.append_text("not matched: ").append_description_of(item) + return False + return True + + def ismatched(self, item: T) -> bool: + for index, matcher in enumerate(self.matchers): + if matcher.matches(item): + del self.matchers[index] + return True + + if self.mismatch_description: + self.mismatch_description.append_text("not matched: ").append_description_of(item) + return False + + +class IsSequenceContainingInAnyOrder(BaseMatcher[Sequence[T]]): + def __init__(self, matchers: Sequence[Matcher[T]]) -> None: + self.matchers = matchers + + def matches( + self, item: Sequence[T], mismatch_description: Optional[Description] = None + ) -> bool: + try: + sequence = list(item) + matchsequence = MatchInAnyOrder(self.matchers, mismatch_description) + for element in sequence: + if not matchsequence.matches(element): + return False + return matchsequence.isfinished(sequence) + except TypeError: + if mismatch_description: + super(IsSequenceContainingInAnyOrder, self).describe_mismatch( + item, mismatch_description + ) + return False + + def describe_mismatch(self, item: Sequence[T], mismatch_description: Description) -> None: + self.matches(item, mismatch_description) + + def describe_to(self, description: Description) -> None: + description.append_text("a sequence over ").append_list( + "[", ", ", "]", self.matchers + ).append_text(" in any order") + + +def contains_inanyorder(*items: Union[Matcher[T], T]) -> Matcher[Sequence[T]]: + """Matches if sequences's elements, in any order, satisfy a given list of + matchers. + + :param match1,...: A comma-separated list of matchers. + + This matcher iterates the evaluated sequence, seeing if each element + satisfies any of the given matchers. The matchers are tried from left to + right, and when a satisfied matcher is found, it is no longer a candidate + for the remaining elements. If a one-to-one correspondence is established + between elements and matchers, ``contains_inanyorder`` is satisfied. + + Any argument that is not a matcher is implicitly wrapped in an + :py:func:`~hamcrest.core.core.isequal.equal_to` matcher to check for + equality. + + """ + + matchers = [] + for item in items: + matchers.append(wrap_matcher(item)) + return IsSequenceContainingInAnyOrder(matchers) diff --git a/WebCrawler/venv/Lib/site-packages/hamcrest/library/collection/issequence_containinginorder.py b/WebCrawler/venv/Lib/site-packages/hamcrest/library/collection/issequence_containinginorder.py new file mode 100644 index 0000000..c7cbda3 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/hamcrest/library/collection/issequence_containinginorder.py @@ -0,0 +1,103 @@ +import warnings +from typing import Optional, Sequence, TypeVar, Union + +from hamcrest.core.base_matcher import BaseMatcher +from hamcrest.core.description import Description +from hamcrest.core.helpers.wrap_matcher import wrap_matcher +from hamcrest.core.matcher import Matcher + +__author__ = "Jon Reid" +__copyright__ = "Copyright 2011 hamcrest.org" +__license__ = "BSD, see License.txt" + +T = TypeVar("T") + + +class MatchingInOrder(object): + def __init__( + self, matchers: Sequence[Matcher[T]], mismatch_description: Optional[Description] + ) -> None: + self.matchers = matchers + self.mismatch_description = mismatch_description + self.next_match_index = 0 + + def matches(self, item: T) -> bool: + return self.isnotsurplus(item) and self.ismatched(item) + + def isfinished(self) -> bool: + if self.next_match_index < len(self.matchers): + if self.mismatch_description: + self.mismatch_description.append_text("No item matched: ").append_description_of( + self.matchers[self.next_match_index] + ) + return False + return True + + def ismatched(self, item: T) -> bool: + matcher = self.matchers[self.next_match_index] + if not matcher.matches(item): + if self.mismatch_description: + self.mismatch_description.append_text("item " + str(self.next_match_index) + ": ") + matcher.describe_mismatch(item, self.mismatch_description) + return False + self.next_match_index += 1 + return True + + def isnotsurplus(self, item: T) -> bool: + if len(self.matchers) <= self.next_match_index: + if self.mismatch_description: + self.mismatch_description.append_text("Not matched: ").append_description_of(item) + return False + return True + + +class IsSequenceContainingInOrder(BaseMatcher[Sequence[T]]): + def __init__(self, matchers: Sequence[Matcher[T]]) -> None: + self.matchers = matchers + + def matches( + self, item: Sequence[T], mismatch_description: Optional[Description] = None + ) -> bool: + try: + matchsequence = MatchingInOrder(self.matchers, mismatch_description) + for element in item: + if not matchsequence.matches(element): + return False + return matchsequence.isfinished() + except TypeError: + if mismatch_description: + super(IsSequenceContainingInOrder, self).describe_mismatch( + item, mismatch_description + ) + return False + + def describe_mismatch(self, item: Sequence[T], mismatch_description: Description) -> None: + self.matches(item, mismatch_description) + + def describe_to(self, description: Description) -> None: + description.append_text("a sequence containing ").append_list("[", ", ", "]", self.matchers) + + +def contains_exactly(*items: Union[Matcher[T], T]) -> Matcher[Sequence[T]]: + """Matches if sequence's elements satisfy a given list of matchers, in order. + + :param match1,...: A comma-separated list of matchers. + + This matcher iterates the evaluated sequence and a given list of matchers, + seeing if each element satisfies its corresponding matcher. + + Any argument that is not a matcher is implicitly wrapped in an + :py:func:`~hamcrest.core.core.isequal.equal_to` matcher to check for + equality. + + """ + matchers = [] + for item in items: + matchers.append(wrap_matcher(item)) + return IsSequenceContainingInOrder(matchers) + + +def contains(*items: Union[Matcher[T], T]) -> Matcher[Sequence[T]]: + """Deprecated - use contains_exactly(*items)""" + warnings.warn("deprecated - use contains_exactly(*items)", DeprecationWarning) + return contains_exactly(*items) diff --git a/WebCrawler/venv/Lib/site-packages/hamcrest/library/collection/issequence_onlycontaining.py b/WebCrawler/venv/Lib/site-packages/hamcrest/library/collection/issequence_onlycontaining.py new file mode 100644 index 0000000..f781998 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/hamcrest/library/collection/issequence_onlycontaining.py @@ -0,0 +1,60 @@ +from typing import Sequence, TypeVar, Union + +from hamcrest.core.base_matcher import BaseMatcher +from hamcrest.core.core.anyof import any_of +from hamcrest.core.description import Description +from hamcrest.core.helpers.wrap_matcher import wrap_matcher +from hamcrest.core.matcher import Matcher + +__author__ = "Jon Reid" +__copyright__ = "Copyright 2011 hamcrest.org" +__license__ = "BSD, see License.txt" + +T = TypeVar("T") + + +class IsSequenceOnlyContaining(BaseMatcher[Sequence[T]]): + def __init__(self, matcher: Matcher[T]) -> None: + self.matcher = matcher + + def _matches(self, item: Sequence[T]) -> bool: + try: + sequence = list(item) + if len(sequence) == 0: + return False + for element in sequence: + if not self.matcher.matches(element): + return False + return True + except TypeError: + return False + + def describe_to(self, description: Description) -> None: + description.append_text("a sequence containing items matching ").append_description_of( + self.matcher + ) + + +def only_contains(*items: Union[Matcher[T], T]) -> Matcher[Sequence[T]]: + """Matches if each element of sequence satisfies any of the given matchers. + + :param match1,...: A comma-separated list of matchers. + + This matcher iterates the evaluated sequence, confirming whether each + element satisfies any of the given matchers. + + Example:: + + only_contains(less_than(4)) + + will match ``[3,1,2]``. + + Any argument that is not a matcher is implicitly wrapped in an + :py:func:`~hamcrest.core.core.isequal.equal_to` matcher to check for + equality. + + """ + matchers = [] + for item in items: + matchers.append(wrap_matcher(item)) + return IsSequenceOnlyContaining(any_of(*matchers)) diff --git a/WebCrawler/venv/Lib/site-packages/hamcrest/library/integration/__init__.py b/WebCrawler/venv/Lib/site-packages/hamcrest/library/integration/__init__.py new file mode 100644 index 0000000..b28371c --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/hamcrest/library/integration/__init__.py @@ -0,0 +1,7 @@ +"""Utilities for integrating Hamcrest with other libraries.""" + +from .match_equality import match_equality + +__author__ = "Jon Reid" +__copyright__ = "Copyright 2011 hamcrest.org" +__license__ = "BSD, see License.txt" diff --git a/WebCrawler/venv/Lib/site-packages/hamcrest/library/integration/__pycache__/__init__.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/hamcrest/library/integration/__pycache__/__init__.cpython-39.pyc new file mode 100644 index 0000000..72fde77 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/hamcrest/library/integration/__pycache__/__init__.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/hamcrest/library/integration/__pycache__/match_equality.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/hamcrest/library/integration/__pycache__/match_equality.cpython-39.pyc new file mode 100644 index 0000000..9e8e859 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/hamcrest/library/integration/__pycache__/match_equality.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/hamcrest/library/integration/match_equality.py b/WebCrawler/venv/Lib/site-packages/hamcrest/library/integration/match_equality.py new file mode 100644 index 0000000..33494ef --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/hamcrest/library/integration/match_equality.py @@ -0,0 +1,44 @@ +from typing import Any + +from hamcrest.core.helpers.wrap_matcher import wrap_matcher +from hamcrest.core.matcher import Matcher +from hamcrest.core.string_description import tostring + +__author__ = "Chris Rose" +__copyright__ = "Copyright 2011 hamcrest.org" +__license__ = "BSD, see License.txt" +__unittest = True + + +class EqualityWrapper(object): + def __init__(self, matcher: Matcher) -> None: + self.matcher = matcher + + def __eq__(self, obj: Any) -> bool: + return self.matcher.matches(obj) + + def __str__(self) -> str: + return repr(self) + + def __repr__(self) -> str: + return tostring(self.matcher) + + +def match_equality(matcher: Matcher) -> EqualityWrapper: + """Wraps a matcher to define equality in terms of satisfying the matcher. + + ``match_equality`` allows Hamcrest matchers to be used in libraries that + are not Hamcrest-aware. They might use the equality operator:: + + assert match_equality(matcher) == object + + Or they might provide a method that uses equality for its test:: + + library.method_that_tests_eq(match_equality(matcher)) + + One concrete example is integrating with the ``assert_called_with`` methods + in Michael Foord's `mock `_ + library. + + """ + return EqualityWrapper(wrap_matcher(matcher)) diff --git a/WebCrawler/venv/Lib/site-packages/hamcrest/library/number/__init__.py b/WebCrawler/venv/Lib/site-packages/hamcrest/library/number/__init__.py new file mode 100644 index 0000000..ca9b151 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/hamcrest/library/number/__init__.py @@ -0,0 +1,13 @@ +"""Matchers that perform numeric comparisons.""" + +from .iscloseto import close_to +from .ordering_comparison import ( + greater_than, + greater_than_or_equal_to, + less_than, + less_than_or_equal_to, +) + +__author__ = "Jon Reid" +__copyright__ = "Copyright 2011 hamcrest.org" +__license__ = "BSD, see License.txt" diff --git a/WebCrawler/venv/Lib/site-packages/hamcrest/library/number/__pycache__/__init__.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/hamcrest/library/number/__pycache__/__init__.cpython-39.pyc new file mode 100644 index 0000000..29d3fac Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/hamcrest/library/number/__pycache__/__init__.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/hamcrest/library/number/__pycache__/iscloseto.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/hamcrest/library/number/__pycache__/iscloseto.cpython-39.pyc new file mode 100644 index 0000000..86d729a Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/hamcrest/library/number/__pycache__/iscloseto.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/hamcrest/library/number/__pycache__/ordering_comparison.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/hamcrest/library/number/__pycache__/ordering_comparison.cpython-39.pyc new file mode 100644 index 0000000..cda58fe Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/hamcrest/library/number/__pycache__/ordering_comparison.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/hamcrest/library/number/iscloseto.py b/WebCrawler/venv/Lib/site-packages/hamcrest/library/number/iscloseto.py new file mode 100644 index 0000000..69ff025 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/hamcrest/library/number/iscloseto.py @@ -0,0 +1,91 @@ +from decimal import Decimal +from math import fabs +from typing import Any, Union, overload + +from hamcrest.core.base_matcher import BaseMatcher +from hamcrest.core.description import Description +from hamcrest.core.matcher import Matcher + +__author__ = "Jon Reid" +__copyright__ = "Copyright 2011 hamcrest.org" +__license__ = "BSD, see License.txt" + +Number = Union[float, Decimal] # Argh, https://github.com/python/mypy/issues/3186 + + +def isnumeric(value: Any) -> bool: + """Confirm that 'value' can be treated numerically; duck-test accordingly + """ + if isinstance(value, (float, complex, int)): + return True + + try: + _ = (fabs(value) + 0 - 0) * 1 + return True + except ArithmeticError: + return True + except: + return False + + +class IsCloseTo(BaseMatcher[Number]): + def __init__(self, value: Number, delta: Number) -> None: + if not isnumeric(value): + raise TypeError("IsCloseTo value must be numeric") + if not isnumeric(delta): + raise TypeError("IsCloseTo delta must be numeric") + + self.value = value + self.delta = delta + + def _matches(self, item: Number) -> bool: + if not isnumeric(item): + return False + return self._diff(item) <= self.delta + + def _diff(self, item: Number) -> float: + # TODO - Fails for mixed floats & Decimals + return fabs(item - self.value) # type: ignore + + def describe_mismatch(self, item: Number, mismatch_description: Description) -> None: + if not isnumeric(item): + super(IsCloseTo, self).describe_mismatch(item, mismatch_description) + else: + actual_delta = self._diff(item) + mismatch_description.append_description_of(item).append_text( + " differed by " + ).append_description_of(actual_delta) + + def describe_to(self, description: Description) -> None: + description.append_text("a numeric value within ").append_description_of( + self.delta + ).append_text(" of ").append_description_of(self.value) + + +@overload +def close_to(value: float, delta: float) -> Matcher[float]: + ... + + +@overload +def close_to(value: Decimal, delta: Decimal) -> Matcher[Decimal]: + ... + + +def close_to(value, delta): + """Matches if object is a number close to a given value, within a given + delta. + + :param value: The value to compare against as the expected value. + :param delta: The maximum delta between the values for which the numbers + are considered close. + + This matcher compares the evaluated object against ``value`` to see if the + difference is within a positive ``delta``. + + Example:: + + close_to(3.0, 0.25) + + """ + return IsCloseTo(value, delta) diff --git a/WebCrawler/venv/Lib/site-packages/hamcrest/library/number/ordering_comparison.py b/WebCrawler/venv/Lib/site-packages/hamcrest/library/number/ordering_comparison.py new file mode 100644 index 0000000..6c6275d --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/hamcrest/library/number/ordering_comparison.py @@ -0,0 +1,66 @@ +import operator +from typing import Any, Callable + +from hamcrest.core.base_matcher import BaseMatcher +from hamcrest.core.description import Description +from hamcrest.core.matcher import Matcher + +__author__ = "Jon Reid" +__copyright__ = "Copyright 2011 hamcrest.org" +__license__ = "BSD, see License.txt" + + +class OrderingComparison(BaseMatcher[Any]): + def __init__( + self, + value: Any, + comparison_function: Callable[[Any, Any], bool], + comparison_description: str, + ) -> None: + self.value = value + self.comparison_function = comparison_function + self.comparison_description = comparison_description + + def _matches(self, item: Any) -> bool: + return self.comparison_function(item, self.value) + + def describe_to(self, description: Description) -> None: + description.append_text("a value ").append_text(self.comparison_description).append_text( + " " + ).append_description_of(self.value) + + +def greater_than(value: Any) -> Matcher[Any]: + """Matches if object is greater than a given value. + + :param value: The value to compare against. + + """ + return OrderingComparison(value, operator.gt, "greater than") + + +def greater_than_or_equal_to(value: Any) -> Matcher[Any]: + """Matches if object is greater than or equal to a given value. + + :param value: The value to compare against. + + """ + return OrderingComparison(value, operator.ge, "greater than or equal to") + + +def less_than(value: Any) -> Matcher[Any]: + """Matches if object is less than a given value. + + :param value: The value to compare against. + + """ + return OrderingComparison(value, operator.lt, "less than") + + +def less_than_or_equal_to(value: Any) -> Matcher[Any]: + """Matches if object is less than or equal to a given value. + + :param value: The value to compare against. + + """ + return OrderingComparison(value, operator.le, "less than or equal to") diff --git a/WebCrawler/venv/Lib/site-packages/hamcrest/library/object/__init__.py b/WebCrawler/venv/Lib/site-packages/hamcrest/library/object/__init__.py new file mode 100644 index 0000000..af84d7e --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/hamcrest/library/object/__init__.py @@ -0,0 +1,9 @@ +"""Matchers that inspect objects and classes.""" + +from .haslength import has_length +from .hasproperty import has_properties, has_property +from .hasstring import has_string + +__author__ = "Jon Reid" +__copyright__ = "Copyright 2011 hamcrest.org" +__license__ = "BSD, see License.txt" diff --git a/WebCrawler/venv/Lib/site-packages/hamcrest/library/object/__pycache__/__init__.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/hamcrest/library/object/__pycache__/__init__.cpython-39.pyc new file mode 100644 index 0000000..a49bfe8 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/hamcrest/library/object/__pycache__/__init__.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/hamcrest/library/object/__pycache__/haslength.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/hamcrest/library/object/__pycache__/haslength.cpython-39.pyc new file mode 100644 index 0000000..65484c4 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/hamcrest/library/object/__pycache__/haslength.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/hamcrest/library/object/__pycache__/hasproperty.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/hamcrest/library/object/__pycache__/hasproperty.cpython-39.pyc new file mode 100644 index 0000000..8b35e52 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/hamcrest/library/object/__pycache__/hasproperty.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/hamcrest/library/object/__pycache__/hasstring.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/hamcrest/library/object/__pycache__/hasstring.cpython-39.pyc new file mode 100644 index 0000000..3b97a8d Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/hamcrest/library/object/__pycache__/hasstring.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/hamcrest/library/object/haslength.py b/WebCrawler/venv/Lib/site-packages/hamcrest/library/object/haslength.py new file mode 100644 index 0000000..86d0399 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/hamcrest/library/object/haslength.py @@ -0,0 +1,52 @@ +from collections.abc import Sized +from typing import Union + +from hamcrest.core.base_matcher import BaseMatcher +from hamcrest.core.description import Description +from hamcrest.core.helpers.hasmethod import hasmethod +from hamcrest.core.helpers.wrap_matcher import wrap_matcher +from hamcrest.core.matcher import Matcher + +__author__ = "Jon Reid" +__copyright__ = "Copyright 2011 hamcrest.org" +__license__ = "BSD, see License.txt" + + +class HasLength(BaseMatcher[Sized]): + def __init__(self, len_matcher: Matcher[int]) -> None: + self.len_matcher = len_matcher + + def _matches(self, item: Sized) -> bool: + if not hasmethod(item, "__len__"): + return False + return self.len_matcher.matches(len(item)) + + def describe_mismatch(self, item: Sized, mismatch_description: Description) -> None: + super(HasLength, self).describe_mismatch(item, mismatch_description) + if hasmethod(item, "__len__"): + mismatch_description.append_text(" with length of ").append_description_of(len(item)) + + def describe_to(self, description: Description) -> None: + description.append_text("an object with length of ").append_description_of(self.len_matcher) + + +def has_length(match: Union[int, Matcher[int]]) -> Matcher[Sized]: + """Matches if ``len(item)`` satisfies a given matcher. + + :param match: The matcher to satisfy, or an expected value for + :py:func:`~hamcrest.core.core.isequal.equal_to` matching. + + This matcher invokes the :py:func:`len` function on the evaluated object to + get its length, passing the result to a given matcher for evaluation. + + If the ``match`` argument is not a matcher, it is implicitly wrapped in an + :py:func:`~hamcrest.core.core.isequal.equal_to` matcher to check for + :equality. + + Examples:: + + has_length(greater_than(6)) + has_length(5) + + """ + return HasLength(wrap_matcher(match)) diff --git a/WebCrawler/venv/Lib/site-packages/hamcrest/library/object/hasproperty.py b/WebCrawler/venv/Lib/site-packages/hamcrest/library/object/hasproperty.py new file mode 100644 index 0000000..b27036d --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/hamcrest/library/object/hasproperty.py @@ -0,0 +1,202 @@ +from typing import Any, Mapping, TypeVar, Union, overload + +from hamcrest import described_as +from hamcrest.core import anything +from hamcrest.core.base_matcher import BaseMatcher +from hamcrest.core.core.allof import AllOf +from hamcrest.core.description import Description +from hamcrest.core.helpers.wrap_matcher import wrap_matcher as wrap_shortcut +from hamcrest.core.matcher import Matcher +from hamcrest.core.string_description import StringDescription + +__author__ = "Chris Rose" +__copyright__ = "Copyright 2011 hamcrest.org" +__license__ = "BSD, see License.txt" + +V = TypeVar("V") + + +class IsObjectWithProperty(BaseMatcher[object]): + def __init__(self, property_name: str, value_matcher: Matcher[V]) -> None: + self.property_name = property_name + self.value_matcher = value_matcher + + def _matches(self, item: object) -> bool: + if item is None: + return False + + if not hasattr(item, self.property_name): + return False + + value = getattr(item, self.property_name) + return self.value_matcher.matches(value) + + def describe_to(self, description: Description) -> None: + description.append_text("an object with a property '").append_text( + self.property_name + ).append_text("' matching ").append_description_of(self.value_matcher) + + def describe_mismatch(self, item: object, mismatch_description: Description) -> None: + if item is None: + mismatch_description.append_text("was None") + return + + if not hasattr(item, self.property_name): + mismatch_description.append_description_of(item).append_text( + " did not have the " + ).append_description_of(self.property_name).append_text(" property") + return + + mismatch_description.append_text("property ").append_description_of( + self.property_name + ).append_text(" ") + value = getattr(item, self.property_name) + self.value_matcher.describe_mismatch(value, mismatch_description) + + def __str__(self): + d = StringDescription() + self.describe_to(d) + return str(d) + + +def has_property(name: str, match: Union[None, Matcher[V], V] = None) -> Matcher[object]: + """Matches if object has a property with a given name whose value satisfies + a given matcher. + + :param name: The name of the property. + :param match: Optional matcher to satisfy. + + This matcher determines if the evaluated object has a property with a given + name. If no such property is found, ``has_property`` is not satisfied. + + If the property is found, its value is passed to a given matcher for + evaluation. If the ``match`` argument is not a matcher, it is implicitly + wrapped in an :py:func:`~hamcrest.core.core.isequal.equal_to` matcher to + check for equality. + + If the ``match`` argument is not provided, the + :py:func:`~hamcrest.core.core.isanything.anything` matcher is used so that + ``has_property`` is satisfied if a matching property is found. + + Examples:: + + has_property('name', starts_with('J')) + has_property('name', 'Jon') + has_property('name') + + """ + + if match is None: + match = anything() + + return IsObjectWithProperty(name, wrap_shortcut(match)) + + +# Keyword argument form +@overload +def has_properties(**keys_valuematchers: Union[Matcher[V], V]) -> Matcher[object]: + ... + + +# Name to matcher dict form +@overload +def has_properties(keys_valuematchers: Mapping[str, Union[Matcher[V], V]]) -> Matcher[object]: + ... + + +# Alternating name/matcher form +@overload +def has_properties(*keys_valuematchers: Any) -> Matcher[object]: + ... + + +def has_properties(*keys_valuematchers, **kv_args): + """Matches if an object has properties satisfying all of a dictionary + of string property names and corresponding value matchers. + + :param matcher_dict: A dictionary mapping keys to associated value matchers, + or to expected values for + :py:func:`~hamcrest.core.core.isequal.equal_to` matching. + + Note that the keys must be actual keys, not matchers. Any value argument + that is not a matcher is implicitly wrapped in an + :py:func:`~hamcrest.core.core.isequal.equal_to` matcher to check for + equality. + + Examples:: + + has_properties({'foo':equal_to(1), 'bar':equal_to(2)}) + has_properties({'foo':1, 'bar':2}) + + ``has_properties`` also accepts a list of keyword arguments: + + .. function:: has_properties(keyword1=value_matcher1[, keyword2=value_matcher2[, ...]]) + + :param keyword1: A keyword to look up. + :param valueMatcher1: The matcher to satisfy for the value, or an expected + value for :py:func:`~hamcrest.core.core.isequal.equal_to` matching. + + Examples:: + + has_properties(foo=equal_to(1), bar=equal_to(2)) + has_properties(foo=1, bar=2) + + Finally, ``has_properties`` also accepts a list of alternating keys and their + value matchers: + + .. function:: has_properties(key1, value_matcher1[, ...]) + + :param key1: A key (not a matcher) to look up. + :param valueMatcher1: The matcher to satisfy for the value, or an expected + value for :py:func:`~hamcrest.core.core.isequal.equal_to` matching. + + Examples:: + + has_properties('foo', equal_to(1), 'bar', equal_to(2)) + has_properties('foo', 1, 'bar', 2) + + """ + if len(keys_valuematchers) == 1: + try: + base_dict = keys_valuematchers[0].copy() + for key in base_dict: + base_dict[key] = wrap_shortcut(base_dict[key]) + except AttributeError: + raise ValueError( + "single-argument calls to has_properties must pass a dict as the argument" + ) + else: + if len(keys_valuematchers) % 2: + raise ValueError("has_properties requires key-value pairs") + base_dict = {} + for index in range(int(len(keys_valuematchers) / 2)): + base_dict[keys_valuematchers[2 * index]] = wrap_shortcut( + keys_valuematchers[2 * index + 1] + ) + + for key, value in kv_args.items(): + base_dict[key] = wrap_shortcut(value) + + if len(base_dict) > 1: + description = StringDescription().append_text("an object with properties ") + for i, (property_name, property_value_matcher) in enumerate(sorted(base_dict.items())): + description.append_description_of(property_name).append_text( + " matching " + ).append_description_of(property_value_matcher) + if i < len(base_dict) - 1: + description.append_text(" and ") + + return described_as( + str(description), + AllOf( + *[ + has_property(property_name, property_value_matcher) + for property_name, property_value_matcher in sorted(base_dict.items()) + ], + describe_all_mismatches=True, + describe_matcher_in_mismatch=False, + ), + ) + else: + property_name, property_value_matcher = base_dict.popitem() + return has_property(property_name, property_value_matcher) diff --git a/WebCrawler/venv/Lib/site-packages/hamcrest/library/object/hasstring.py b/WebCrawler/venv/Lib/site-packages/hamcrest/library/object/hasstring.py new file mode 100644 index 0000000..b03d466 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/hamcrest/library/object/hasstring.py @@ -0,0 +1,40 @@ +from hamcrest.core.base_matcher import BaseMatcher +from hamcrest.core.description import Description +from hamcrest.core.helpers.wrap_matcher import wrap_matcher +from hamcrest.core.matcher import Matcher + +__author__ = "Jon Reid" +__copyright__ = "Copyright 2011 hamcrest.org" +__license__ = "BSD, see License.txt" + + +class HasString(BaseMatcher[object]): + def __init__(self, str_matcher: Matcher[str]) -> None: + self.str_matcher = str_matcher + + def _matches(self, item: object) -> bool: + return self.str_matcher.matches(str(item)) + + def describe_to(self, description: Description) -> None: + description.append_text("an object with str ").append_description_of(self.str_matcher) + + +def has_string(match) -> Matcher[object]: + """Matches if ``str(item)`` satisfies a given matcher. + + :param match: The matcher to satisfy, or an expected value for + :py:func:`~hamcrest.core.core.isequal.equal_to` matching. + + This matcher invokes the :py:func:`str` function on the evaluated object to + get its length, passing the result to a given matcher for evaluation. If + the ``match`` argument is not a matcher, it is implicitly wrapped in an + :py:func:`~hamcrest.core.core.isequal.equal_to` matcher to check for + equality. + + Examples:: + + has_string(starts_with('foo')) + has_string('bar') + + """ + return HasString(wrap_matcher(match)) diff --git a/WebCrawler/venv/Lib/site-packages/hamcrest/library/text/__init__.py b/WebCrawler/venv/Lib/site-packages/hamcrest/library/text/__init__.py new file mode 100644 index 0000000..291ac31 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/hamcrest/library/text/__init__.py @@ -0,0 +1,13 @@ +"""Matchers that perform text comparisons.""" + +from .isequal_ignoring_case import equal_to_ignoring_case +from .isequal_ignoring_whitespace import equal_to_ignoring_whitespace +from .stringcontains import contains_string +from .stringcontainsinorder import string_contains_in_order +from .stringendswith import ends_with +from .stringmatches import matches_regexp +from .stringstartswith import starts_with + +__author__ = "Jon Reid" +__copyright__ = "Copyright 2011 hamcrest.org" +__license__ = "BSD, see License.txt" diff --git a/WebCrawler/venv/Lib/site-packages/hamcrest/library/text/__pycache__/__init__.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/hamcrest/library/text/__pycache__/__init__.cpython-39.pyc new file mode 100644 index 0000000..8c63b41 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/hamcrest/library/text/__pycache__/__init__.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/hamcrest/library/text/__pycache__/isequal_ignoring_case.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/hamcrest/library/text/__pycache__/isequal_ignoring_case.cpython-39.pyc new file mode 100644 index 0000000..6fec0c6 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/hamcrest/library/text/__pycache__/isequal_ignoring_case.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/hamcrest/library/text/__pycache__/isequal_ignoring_whitespace.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/hamcrest/library/text/__pycache__/isequal_ignoring_whitespace.cpython-39.pyc new file mode 100644 index 0000000..bfa420a Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/hamcrest/library/text/__pycache__/isequal_ignoring_whitespace.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/hamcrest/library/text/__pycache__/stringcontains.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/hamcrest/library/text/__pycache__/stringcontains.cpython-39.pyc new file mode 100644 index 0000000..d86b2f7 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/hamcrest/library/text/__pycache__/stringcontains.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/hamcrest/library/text/__pycache__/stringcontainsinorder.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/hamcrest/library/text/__pycache__/stringcontainsinorder.cpython-39.pyc new file mode 100644 index 0000000..1131ec9 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/hamcrest/library/text/__pycache__/stringcontainsinorder.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/hamcrest/library/text/__pycache__/stringendswith.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/hamcrest/library/text/__pycache__/stringendswith.cpython-39.pyc new file mode 100644 index 0000000..5998463 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/hamcrest/library/text/__pycache__/stringendswith.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/hamcrest/library/text/__pycache__/stringmatches.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/hamcrest/library/text/__pycache__/stringmatches.cpython-39.pyc new file mode 100644 index 0000000..229ca92 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/hamcrest/library/text/__pycache__/stringmatches.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/hamcrest/library/text/__pycache__/stringstartswith.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/hamcrest/library/text/__pycache__/stringstartswith.cpython-39.pyc new file mode 100644 index 0000000..456a530 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/hamcrest/library/text/__pycache__/stringstartswith.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/hamcrest/library/text/__pycache__/substringmatcher.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/hamcrest/library/text/__pycache__/substringmatcher.cpython-39.pyc new file mode 100644 index 0000000..02b282b Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/hamcrest/library/text/__pycache__/substringmatcher.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/hamcrest/library/text/isequal_ignoring_case.py b/WebCrawler/venv/Lib/site-packages/hamcrest/library/text/isequal_ignoring_case.py new file mode 100644 index 0000000..c7afb18 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/hamcrest/library/text/isequal_ignoring_case.py @@ -0,0 +1,42 @@ +from hamcrest.core.base_matcher import BaseMatcher +from hamcrest.core.description import Description +from hamcrest.core.matcher import Matcher + +__author__ = "Jon Reid" +__copyright__ = "Copyright 2011 hamcrest.org" +__license__ = "BSD, see License.txt" + + +class IsEqualIgnoringCase(BaseMatcher[str]): + def __init__(self, string: str) -> None: + if not isinstance(string, str): + raise TypeError("IsEqualIgnoringCase requires string") + self.original_string = string + self.lowered_string = string.lower() + + def _matches(self, item: str) -> bool: + if not isinstance(item, str): + return False + return self.lowered_string == item.lower() + + def describe_to(self, description: Description) -> None: + description.append_description_of(self.original_string).append_text(" ignoring case") + + +def equal_to_ignoring_case(string: str) -> Matcher[str]: + """Matches if object is a string equal to a given string, ignoring case + differences. + + :param string: The string to compare against as the expected value. + + This matcher first checks whether the evaluated object is a string. If so, + it compares it with ``string``, ignoring differences of case. + + Example:: + + equal_to_ignoring_case("hello world") + + will match "heLLo WorlD". + + """ + return IsEqualIgnoringCase(string) diff --git a/WebCrawler/venv/Lib/site-packages/hamcrest/library/text/isequal_ignoring_whitespace.py b/WebCrawler/venv/Lib/site-packages/hamcrest/library/text/isequal_ignoring_whitespace.py new file mode 100644 index 0000000..dcb7e85 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/hamcrest/library/text/isequal_ignoring_whitespace.py @@ -0,0 +1,56 @@ +from hamcrest.core.base_matcher import BaseMatcher +from hamcrest.core.description import Description +from hamcrest.core.matcher import Matcher + +__author__ = "Jon Reid" +__copyright__ = "Copyright 2011 hamcrest.org" +__license__ = "BSD, see License.txt" + + +def stripspace(string: str) -> str: + result = "" + last_was_space = True + for character in string: + if character.isspace(): + if not last_was_space: + result += " " + last_was_space = True + else: + result += character + last_was_space = False + return result.strip() + + +class IsEqualIgnoringWhiteSpace(BaseMatcher[str]): + def __init__(self, string) -> None: + if not isinstance(string, str): + raise TypeError("IsEqualIgnoringWhiteSpace requires string") + self.original_string = string + self.stripped_string = stripspace(string) + + def _matches(self, item: str) -> bool: + if not isinstance(item, str): + return False + return self.stripped_string == stripspace(item) + + def describe_to(self, description: Description) -> None: + description.append_description_of(self.original_string).append_text(" ignoring whitespace") + + +def equal_to_ignoring_whitespace(string: str) -> Matcher[str]: + """Matches if object is a string equal to a given string, ignoring + differences in whitespace. + + :param string: The string to compare against as the expected value. + + This matcher first checks whether the evaluated object is a string. If so, + it compares it with ``string``, ignoring differences in runs of whitespace. + + Example:: + + equal_to_ignoring_whitespace("hello world") + + will match ``"hello world"``. + + """ + return IsEqualIgnoringWhiteSpace(string) diff --git a/WebCrawler/venv/Lib/site-packages/hamcrest/library/text/stringcontains.py b/WebCrawler/venv/Lib/site-packages/hamcrest/library/text/stringcontains.py new file mode 100644 index 0000000..34df2f6 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/hamcrest/library/text/stringcontains.py @@ -0,0 +1,38 @@ +from hamcrest.core.helpers.hasmethod import hasmethod +from hamcrest.core.matcher import Matcher +from hamcrest.library.text.substringmatcher import SubstringMatcher + +__author__ = "Jon Reid" +__copyright__ = "Copyright 2011 hamcrest.org" +__license__ = "BSD, see License.txt" + + +class StringContains(SubstringMatcher): + def __init__(self, substring) -> None: + super(StringContains, self).__init__(substring) + + def _matches(self, item: str) -> bool: + if not hasmethod(item, "find"): + return False + return item.find(self.substring) >= 0 + + def relationship(self): + return "containing" + + +def contains_string(substring: str) -> Matcher[str]: + """Matches if object is a string containing a given string. + + :param string: The string to search for. + + This matcher first checks whether the evaluated object is a string. If so, + it checks whether it contains ``string``. + + Example:: + + contains_string("def") + + will match "abcdefg". + + """ + return StringContains(substring) diff --git a/WebCrawler/venv/Lib/site-packages/hamcrest/library/text/stringcontainsinorder.py b/WebCrawler/venv/Lib/site-packages/hamcrest/library/text/stringcontainsinorder.py new file mode 100644 index 0000000..73c61fc --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/hamcrest/library/text/stringcontainsinorder.py @@ -0,0 +1,50 @@ +from hamcrest.core.base_matcher import BaseMatcher +from hamcrest.core.description import Description +from hamcrest.core.helpers.hasmethod import hasmethod +from hamcrest.core.matcher import Matcher + +__author__ = "Romilly Cocking" +__copyright__ = "Copyright 2011 hamcrest.org" +__license__ = "BSD, see License.txt" + + +class StringContainsInOrder(BaseMatcher[str]): + def __init__(self, *substrings) -> None: + for substring in substrings: + if not isinstance(substring, str): + raise TypeError(self.__class__.__name__ + " requires string arguments") + self.substrings = substrings + + def _matches(self, item: str) -> bool: + if not hasmethod(item, "find"): + return False + from_index = 0 + for substring in self.substrings: + from_index = item.find(substring, from_index) + if from_index == -1: + return False + return True + + def describe_to(self, description: Description) -> None: + description.append_list("a string containing ", ", ", " in order", self.substrings) + + +def string_contains_in_order(*substrings: str) -> Matcher[str]: + """Matches if object is a string containing a given list of substrings in + relative order. + + :param string1,...: A comma-separated list of strings. + + This matcher first checks whether the evaluated object is a string. If so, + it checks whether it contains a given list of strings, in relative order to + each other. The searches are performed starting from the beginning of the + evaluated string. + + Example:: + + string_contains_in_order("bc", "fg", "jkl") + + will match "abcdefghijklm". + + """ + return StringContainsInOrder(*substrings) diff --git a/WebCrawler/venv/Lib/site-packages/hamcrest/library/text/stringendswith.py b/WebCrawler/venv/Lib/site-packages/hamcrest/library/text/stringendswith.py new file mode 100644 index 0000000..16bebbd --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/hamcrest/library/text/stringendswith.py @@ -0,0 +1,39 @@ +from hamcrest.core.helpers.hasmethod import hasmethod +from hamcrest.core.matcher import Matcher +from hamcrest.library.text.substringmatcher import SubstringMatcher + +__author__ = "Jon Reid" +__copyright__ = "Copyright 2011 hamcrest.org" +__license__ = "BSD, see License.txt" + + +class StringEndsWith(SubstringMatcher): + def __init__(self, substring) -> None: + super(StringEndsWith, self).__init__(substring) + + def _matches(self, item: str) -> bool: + if not hasmethod(item, "endswith"): + return False + return item.endswith(self.substring) + + def relationship(self): + return "ending with" + + +def ends_with(string: str) -> Matcher[str]: + """Matches if object is a string ending with a given string. + + :param string: The string to search for. + + This matcher first checks whether the evaluated object is a string. If so, + it checks if ``string`` matches the ending characters of the evaluated + object. + + Example:: + + ends_with("bar") + + will match "foobar". + + """ + return StringEndsWith(string) diff --git a/WebCrawler/venv/Lib/site-packages/hamcrest/library/text/stringmatches.py b/WebCrawler/venv/Lib/site-packages/hamcrest/library/text/stringmatches.py new file mode 100644 index 0000000..e7e8dd4 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/hamcrest/library/text/stringmatches.py @@ -0,0 +1,40 @@ +import re +from typing import Pattern, Union + +from hamcrest.core.base_matcher import BaseMatcher +from hamcrest.core.description import Description +from hamcrest.core.matcher import Matcher + +__author__ = "Chris Rose" +__copyright__ = "Copyright 2011 hamcrest.org" +__license__ = "BSD, see License.txt" + + +class StringMatchesPattern(BaseMatcher[str]): + def __init__(self, pattern) -> None: + self.pattern = pattern + + def describe_to(self, description: Description) -> None: + description.append_text("a string matching '").append_text( + self.pattern.pattern + ).append_text("'") + + def _matches(self, item: str) -> bool: + return self.pattern.search(item) is not None + + +def matches_regexp(pattern: Union[str, Pattern[str]]) -> Matcher[str]: + """Matches if object is a string containing a match for a given regular + expression. + + :param pattern: The regular expression to search for. + + This matcher first checks whether the evaluated object is a string. If so, + it checks if the regular expression ``pattern`` matches anywhere within the + evaluated object. + + """ + if isinstance(pattern, str): + pattern = re.compile(pattern) + + return StringMatchesPattern(pattern) diff --git a/WebCrawler/venv/Lib/site-packages/hamcrest/library/text/stringstartswith.py b/WebCrawler/venv/Lib/site-packages/hamcrest/library/text/stringstartswith.py new file mode 100644 index 0000000..c3cc7ee --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/hamcrest/library/text/stringstartswith.py @@ -0,0 +1,39 @@ +from hamcrest.core.helpers.hasmethod import hasmethod +from hamcrest.core.matcher import Matcher +from hamcrest.library.text.substringmatcher import SubstringMatcher + +__author__ = "Jon Reid" +__copyright__ = "Copyright 2011 hamcrest.org" +__license__ = "BSD, see License.txt" + + +class StringStartsWith(SubstringMatcher): + def __init__(self, substring) -> None: + super(StringStartsWith, self).__init__(substring) + + def _matches(self, item: str) -> bool: + if not hasmethod(item, "startswith"): + return False + return item.startswith(self.substring) + + def relationship(self): + return "starting with" + + +def starts_with(substring: str) -> Matcher[str]: + """Matches if object is a string starting with a given string. + + :param string: The string to search for. + + This matcher first checks whether the evaluated object is a string. If so, + it checks if ``string`` matches the beginning characters of the evaluated + object. + + Example:: + + starts_with("foo") + + will match "foobar". + + """ + return StringStartsWith(substring) diff --git a/WebCrawler/venv/Lib/site-packages/hamcrest/library/text/substringmatcher.py b/WebCrawler/venv/Lib/site-packages/hamcrest/library/text/substringmatcher.py new file mode 100644 index 0000000..ec7874c --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/hamcrest/library/text/substringmatcher.py @@ -0,0 +1,24 @@ +from abc import ABCMeta, abstractmethod + +from hamcrest.core.base_matcher import BaseMatcher +from hamcrest.core.description import Description + +__author__ = "Jon Reid" +__copyright__ = "Copyright 2011 hamcrest.org" +__license__ = "BSD, see License.txt" + + +class SubstringMatcher(BaseMatcher[str], metaclass=ABCMeta): + def __init__(self, substring) -> None: + if not isinstance(substring, str): + raise TypeError(self.__class__.__name__ + " requires string") + self.substring = substring + + def describe_to(self, description: Description) -> None: + description.append_text("a string ").append_text(self.relationship()).append_text( + " " + ).append_description_of(self.substring) + + @abstractmethod + def relationship(self): + ... diff --git a/WebCrawler/venv/Lib/site-packages/hamcrest/py.typed b/WebCrawler/venv/Lib/site-packages/hamcrest/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/WebCrawler/venv/Lib/site-packages/hyperlink-20.0.1.dist-info/INSTALLER b/WebCrawler/venv/Lib/site-packages/hyperlink-20.0.1.dist-info/INSTALLER new file mode 100644 index 0000000..a1b589e --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/hyperlink-20.0.1.dist-info/INSTALLER @@ -0,0 +1 @@ +pip diff --git a/WebCrawler/venv/Lib/site-packages/hyperlink-20.0.1.dist-info/LICENSE b/WebCrawler/venv/Lib/site-packages/hyperlink-20.0.1.dist-info/LICENSE new file mode 100644 index 0000000..a73f882 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/hyperlink-20.0.1.dist-info/LICENSE @@ -0,0 +1,29 @@ +Copyright (c) 2017 +Glyph Lefkowitz +Itamar Turner-Trauring +Jean Paul Calderone +Adi Roiban +Amber Hawkie Brown +Mahmoud Hashemi +Wilfredo Sanchez Vega + +and others that have contributed code to the public domain. + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/WebCrawler/venv/Lib/site-packages/hyperlink-20.0.1.dist-info/METADATA b/WebCrawler/venv/Lib/site-packages/hyperlink-20.0.1.dist-info/METADATA new file mode 100644 index 0000000..129437c --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/hyperlink-20.0.1.dist-info/METADATA @@ -0,0 +1,37 @@ +Metadata-Version: 2.1 +Name: hyperlink +Version: 20.0.1 +Summary: A featureful, immutable, and correct URL for Python. +Home-page: https://github.com/python-hyper/hyperlink +Author: Mahmoud Hashemi and Glyph Lefkowitz +Author-email: mahmoud@hatnote.com +License: MIT +Platform: any +Classifier: Topic :: Utilities +Classifier: Intended Audience :: Developers +Classifier: Topic :: Software Development :: Libraries +Classifier: Development Status :: 5 - Production/Stable +Classifier: Programming Language :: Python :: 2 +Classifier: Programming Language :: Python :: 2.6 +Classifier: Programming Language :: Python :: 2.7 +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3.4 +Classifier: Programming Language :: Python :: 3.5 +Classifier: Programming Language :: Python :: 3.6 +Classifier: Programming Language :: Python :: 3.7 +Classifier: Programming Language :: Python :: 3.8 +Classifier: Programming Language :: Python :: Implementation :: PyPy +Classifier: License :: OSI Approved :: MIT License +Requires-Python: >=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.* +Requires-Dist: idna (>=2.5) +Requires-Dist: typing ; python_version < "3.5" + +The humble, but powerful, URL runs everything around us. Chances +are you've used several just to read this text. + +Hyperlink is a featureful, pure-Python implementation of the URL, with +an emphasis on correctness. MIT licensed. + +See the docs at http://hyperlink.readthedocs.io. + + diff --git a/WebCrawler/venv/Lib/site-packages/hyperlink-20.0.1.dist-info/RECORD b/WebCrawler/venv/Lib/site-packages/hyperlink-20.0.1.dist-info/RECORD new file mode 100644 index 0000000..7609d1d --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/hyperlink-20.0.1.dist-info/RECORD @@ -0,0 +1,33 @@ +hyperlink-20.0.1.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4 +hyperlink-20.0.1.dist-info/LICENSE,sha256=dGje1VMQ9k-_CFuOiS34HRvMSSY79TXTwsWLAUz64hQ,1231 +hyperlink-20.0.1.dist-info/METADATA,sha256=Br8YFucn2pmP2p3k41wrk5-qySVgrU1SwJjMGYYVhaM,1440 +hyperlink-20.0.1.dist-info/RECORD,, +hyperlink-20.0.1.dist-info/WHEEL,sha256=kGT74LWyRUZrL4VgLh6_g12IeVl_9u9ZVhadrgXZUEY,110 +hyperlink-20.0.1.dist-info/top_level.txt,sha256=qKx9FGU_zxD9mGqiFgleNejfO4AwPY7duhQPaZ30U_M,10 +hyperlink/__init__.py,sha256=d04Ov5_a3TwD4dUCayvMlQmnSc1_owZNk0za_Uz6Wd0,233 +hyperlink/__pycache__/__init__.cpython-39.pyc,, +hyperlink/__pycache__/_socket.cpython-39.pyc,, +hyperlink/__pycache__/_url.cpython-39.pyc,, +hyperlink/__pycache__/hypothesis.cpython-39.pyc,, +hyperlink/_socket.py,sha256=w-FLqQjMCvvhOPOKDL3taDflAImKiGdmogZDFTemMWY,1767 +hyperlink/_url.py,sha256=ED30riZC74QCV1ttuK7f4d-tVxHdJ-X2Tvi_MonYb6g,82725 +hyperlink/hypothesis.py,sha256=NkgYvC0UzsTVPOcRY2z3kXvau2NWMQZkE3zxUPmDbfw,9491 +hyperlink/py.typed,sha256=zS6lDJDXojzFt7_ZHHKc1rxEwJHVgLt1_1AGEexVm3o,49 +hyperlink/test/__init__.py,sha256=C85wqqrAKvwl6Dpk9Ns9SSnbyvInklv5Ju7EzCQkorI,601 +hyperlink/test/__pycache__/__init__.cpython-39.pyc,, +hyperlink/test/__pycache__/common.cpython-39.pyc,, +hyperlink/test/__pycache__/test_common.cpython-39.pyc,, +hyperlink/test/__pycache__/test_decoded_url.cpython-39.pyc,, +hyperlink/test/__pycache__/test_hypothesis.cpython-39.pyc,, +hyperlink/test/__pycache__/test_parse.cpython-39.pyc,, +hyperlink/test/__pycache__/test_scheme_registration.cpython-39.pyc,, +hyperlink/test/__pycache__/test_socket.cpython-39.pyc,, +hyperlink/test/__pycache__/test_url.cpython-39.pyc,, +hyperlink/test/common.py,sha256=wOg1_pwnaSmofKGsQTXctPjNobiinXobVJT4hq312DI,2547 +hyperlink/test/test_common.py,sha256=3h-yWK0XgFstwma8RLsu2t-tC_tTFvQtfKk8_9UgaSg,3693 +hyperlink/test/test_decoded_url.py,sha256=0HJhq2lNhpofQiXxRqErl5LHR8lwEwSIur9xCTvxkZ0,6594 +hyperlink/test/test_hypothesis.py,sha256=LvqlcU0fIQxLhMygdZnrQttc71RgB39f-kSWhBILT0s,7399 +hyperlink/test/test_parse.py,sha256=emizwXl1TczD8iHMyI2qLtbvS99cep8qtUhNbugGa_M,1107 +hyperlink/test/test_scheme_registration.py,sha256=rx93VfiZDU5UYL-lTZBTerB2wa6yWVr7PKKRDGqpdyM,2604 +hyperlink/test/test_socket.py,sha256=_Xlw2vHAY5HMkCUFHHJQf7PZM5gtGJmJoHCVZ0No5nw,1423 +hyperlink/test/test_url.py,sha256=gW2W9QV3Wclfftm4NnnjPhwGakEMaDP4CwygKbPlcu0,54454 diff --git a/WebCrawler/venv/Lib/site-packages/hyperlink-20.0.1.dist-info/WHEEL b/WebCrawler/venv/Lib/site-packages/hyperlink-20.0.1.dist-info/WHEEL new file mode 100644 index 0000000..ef99c6c --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/hyperlink-20.0.1.dist-info/WHEEL @@ -0,0 +1,6 @@ +Wheel-Version: 1.0 +Generator: bdist_wheel (0.34.2) +Root-Is-Purelib: true +Tag: py2-none-any +Tag: py3-none-any + diff --git a/WebCrawler/venv/Lib/site-packages/hyperlink-20.0.1.dist-info/top_level.txt b/WebCrawler/venv/Lib/site-packages/hyperlink-20.0.1.dist-info/top_level.txt new file mode 100644 index 0000000..81722ce --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/hyperlink-20.0.1.dist-info/top_level.txt @@ -0,0 +1 @@ +hyperlink diff --git a/WebCrawler/venv/Lib/site-packages/hyperlink/__init__.py b/WebCrawler/venv/Lib/site-packages/hyperlink/__init__.py new file mode 100644 index 0000000..f680b01 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/hyperlink/__init__.py @@ -0,0 +1,17 @@ +from ._url import ( + parse, + register_scheme, + URL, + EncodedURL, + DecodedURL, + URLParseError, +) + +__all__ = ( + "parse", + "register_scheme", + "URL", + "EncodedURL", + "DecodedURL", + "URLParseError", +) diff --git a/WebCrawler/venv/Lib/site-packages/hyperlink/__pycache__/__init__.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/hyperlink/__pycache__/__init__.cpython-39.pyc new file mode 100644 index 0000000..89a6f26 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/hyperlink/__pycache__/__init__.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/hyperlink/__pycache__/_socket.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/hyperlink/__pycache__/_socket.cpython-39.pyc new file mode 100644 index 0000000..5b53933 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/hyperlink/__pycache__/_socket.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/hyperlink/__pycache__/_url.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/hyperlink/__pycache__/_url.cpython-39.pyc new file mode 100644 index 0000000..9ad24a9 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/hyperlink/__pycache__/_url.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/hyperlink/__pycache__/hypothesis.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/hyperlink/__pycache__/hypothesis.cpython-39.pyc new file mode 100644 index 0000000..d01f11c Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/hyperlink/__pycache__/hypothesis.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/hyperlink/_socket.py b/WebCrawler/venv/Lib/site-packages/hyperlink/_socket.py new file mode 100644 index 0000000..3bcf897 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/hyperlink/_socket.py @@ -0,0 +1,53 @@ +try: + from socket import inet_pton +except ImportError: + from typing import TYPE_CHECKING + + if TYPE_CHECKING: # pragma: no cover + pass + else: + # based on https://gist.github.com/nnemkin/4966028 + # this code only applies on Windows Python 2.7 + import ctypes + import socket + + class SockAddr(ctypes.Structure): + _fields_ = [ + ("sa_family", ctypes.c_short), + ("__pad1", ctypes.c_ushort), + ("ipv4_addr", ctypes.c_byte * 4), + ("ipv6_addr", ctypes.c_byte * 16), + ("__pad2", ctypes.c_ulong), + ] + + WSAStringToAddressA = ctypes.windll.ws2_32.WSAStringToAddressA + WSAAddressToStringA = ctypes.windll.ws2_32.WSAAddressToStringA + + def inet_pton(address_family, ip_string): + # type: (int, str) -> bytes + addr = SockAddr() + ip_string_bytes = ip_string.encode("ascii") + addr.sa_family = address_family + addr_size = ctypes.c_int(ctypes.sizeof(addr)) + + try: + attribute, size = { + socket.AF_INET: ("ipv4_addr", 4), + socket.AF_INET6: ("ipv6_addr", 16), + }[address_family] + except KeyError: + raise socket.error("unknown address family") + + if ( + WSAStringToAddressA( + ip_string_bytes, + address_family, + None, + ctypes.byref(addr), + ctypes.byref(addr_size), + ) + != 0 + ): + raise socket.error(ctypes.FormatError()) + + return ctypes.string_at(getattr(addr, attribute), size) diff --git a/WebCrawler/venv/Lib/site-packages/hyperlink/_url.py b/WebCrawler/venv/Lib/site-packages/hyperlink/_url.py new file mode 100644 index 0000000..4867d58 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/hyperlink/_url.py @@ -0,0 +1,2409 @@ +# -*- coding: utf-8 -*- +u"""Hyperlink provides Pythonic URL parsing, construction, and rendering. + +Usage is straightforward:: + + >>> import hyperlink + >>> url = hyperlink.parse(u'http://github.com/mahmoud/hyperlink?utm_source=docs') + >>> url.host + u'github.com' + >>> secure_url = url.replace(scheme=u'https') + >>> secure_url.get('utm_source')[0] + u'docs' + +Hyperlink's API centers on the :class:`DecodedURL` type, which wraps +the lower-level :class:`URL`, both of which can be returned by the +:func:`parse()` convenience function. + +""" # noqa: E501 + +import re +import sys +import string +import socket +from socket import AF_INET, AF_INET6 + +try: + from socket import AddressFamily +except ImportError: + AddressFamily = int # type: ignore[assignment,misc] +from typing import ( + Any, + Callable, + Dict, + Iterable, + Iterator, + List, + Mapping, + Optional, + Sequence, + Text, + Tuple, + Type, + TypeVar, + Union, + cast, +) +from unicodedata import normalize +from ._socket import inet_pton + +try: + from collections.abc import Mapping as MappingABC +except ImportError: # Python 2 + from collections import Mapping as MappingABC + +from idna import encode as idna_encode, decode as idna_decode + + +PY2 = sys.version_info[0] == 2 +try: + unichr +except NameError: # Py3 + unichr = chr # type: Callable[[int], Text] +NoneType = type(None) # type: Type[None] +QueryPairs = Tuple[Tuple[Text, Optional[Text]], ...] # internal representation +QueryParameters = Union[ + Mapping[Text, Optional[Text]], + QueryPairs, + Sequence[Tuple[Text, Optional[Text]]], +] +T = TypeVar("T") + + +# from boltons.typeutils +def make_sentinel(name="_MISSING", var_name=""): + # type: (str, str) -> object + """Creates and returns a new **instance** of a new class, suitable for + usage as a "sentinel", a kind of singleton often used to indicate + a value is missing when ``None`` is a valid input. + + Args: + name: Name of the Sentinel + var_name: Set this name to the name of the variable in its respective + module enable pickle-ability. + + >>> make_sentinel(var_name='_MISSING') + _MISSING + + The most common use cases here in boltons are as default values + for optional function arguments, partly because of its + less-confusing appearance in automatically generated + documentation. Sentinels also function well as placeholders in queues + and linked lists. + + .. note:: + + By design, additional calls to ``make_sentinel`` with the same + values will not produce equivalent objects. + + >>> make_sentinel('TEST') == make_sentinel('TEST') + False + >>> type(make_sentinel('TEST')) == type(make_sentinel('TEST')) + False + """ + + class Sentinel(object): + def __init__(self): + # type: () -> None + self.name = name + self.var_name = var_name + + def __repr__(self): + # type: () -> str + if self.var_name: + return self.var_name + return "%s(%r)" % (self.__class__.__name__, self.name) + + if var_name: + # superclass type hints don't allow str return type, but it is + # allowed in the docs, hence the ignore[override] below + def __reduce__(self): + # type: () -> str + return self.var_name + + def __nonzero__(self): + # type: () -> bool + return False + + __bool__ = __nonzero__ + + return Sentinel() + + +_unspecified = _UNSET = make_sentinel("_UNSET") # type: Any + + +# RFC 3986 Section 2.3, Unreserved URI Characters +# https://tools.ietf.org/html/rfc3986#section-2.3 +_UNRESERVED_CHARS = frozenset( + "~-._0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ" "abcdefghijklmnopqrstuvwxyz" +) + + +# URL parsing regex (based on RFC 3986 Appendix B, with modifications) +_URL_RE = re.compile( + r"^((?P[^:/?#]+):)?" + r"((?P<_netloc_sep>//)" + r"(?P[^/?#]*))?" + r"(?P[^?#]*)" + r"(\?(?P[^#]*))?" + r"(#(?P.*))?$" +) +_SCHEME_RE = re.compile(r"^[a-zA-Z0-9+-.]*$") +_AUTHORITY_RE = re.compile( + r"^(?:(?P[^@/?#]*)@)?" + r"(?P" + r"(?:\[(?P[^[\]/?#]*)\])" + r"|(?P[^:/?#[\]]*)" + r"|(?P.*?))?" + r"(?::(?P.*))?$" +) + + +_HEX_CHAR_MAP = dict( + [ + ((a + b).encode("ascii"), unichr(int(a + b, 16)).encode("charmap")) + for a in string.hexdigits + for b in string.hexdigits + ] +) +_ASCII_RE = re.compile("([\x00-\x7f]+)") + +# RFC 3986 section 2.2, Reserved Characters +# https://tools.ietf.org/html/rfc3986#section-2.2 +_GEN_DELIMS = frozenset(u":/?#[]@") +_SUB_DELIMS = frozenset(u"!$&'()*+,;=") +_ALL_DELIMS = _GEN_DELIMS | _SUB_DELIMS + +_USERINFO_SAFE = _UNRESERVED_CHARS | _SUB_DELIMS | set(u"%") +_USERINFO_DELIMS = _ALL_DELIMS - _USERINFO_SAFE +_PATH_SAFE = _USERINFO_SAFE | set(u":@") +_PATH_DELIMS = _ALL_DELIMS - _PATH_SAFE +_SCHEMELESS_PATH_SAFE = _PATH_SAFE - set(":") +_SCHEMELESS_PATH_DELIMS = _ALL_DELIMS - _SCHEMELESS_PATH_SAFE +_FRAGMENT_SAFE = _UNRESERVED_CHARS | _PATH_SAFE | set(u"/?") +_FRAGMENT_DELIMS = _ALL_DELIMS - _FRAGMENT_SAFE +_QUERY_VALUE_SAFE = _UNRESERVED_CHARS | _FRAGMENT_SAFE - set(u"&+") +_QUERY_VALUE_DELIMS = _ALL_DELIMS - _QUERY_VALUE_SAFE +_QUERY_KEY_SAFE = _UNRESERVED_CHARS | _QUERY_VALUE_SAFE - set(u"=") +_QUERY_KEY_DELIMS = _ALL_DELIMS - _QUERY_KEY_SAFE + + +def _make_decode_map(delims, allow_percent=False): + # type: (Iterable[Text], bool) -> Mapping[bytes, bytes] + ret = dict(_HEX_CHAR_MAP) + if not allow_percent: + delims = set(delims) | set([u"%"]) + for delim in delims: + _hexord = "{0:02X}".format(ord(delim)).encode("ascii") + _hexord_lower = _hexord.lower() + ret.pop(_hexord) + if _hexord != _hexord_lower: + ret.pop(_hexord_lower) + return ret + + +def _make_quote_map(safe_chars): + # type: (Iterable[Text]) -> Mapping[Union[int, Text], Text] + ret = {} # type: Dict[Union[int, Text], Text] + # v is included in the dict for py3 mostly, because bytestrings + # are iterables of ints, of course! + for i, v in zip(range(256), range(256)): + c = chr(v) + if c in safe_chars: + ret[c] = ret[v] = c + else: + ret[c] = ret[v] = "%{0:02X}".format(i) + return ret + + +_USERINFO_PART_QUOTE_MAP = _make_quote_map(_USERINFO_SAFE) +_USERINFO_DECODE_MAP = _make_decode_map(_USERINFO_DELIMS) +_PATH_PART_QUOTE_MAP = _make_quote_map(_PATH_SAFE) +_SCHEMELESS_PATH_PART_QUOTE_MAP = _make_quote_map(_SCHEMELESS_PATH_SAFE) +_PATH_DECODE_MAP = _make_decode_map(_PATH_DELIMS) +_QUERY_KEY_QUOTE_MAP = _make_quote_map(_QUERY_KEY_SAFE) +_QUERY_KEY_DECODE_MAP = _make_decode_map(_QUERY_KEY_DELIMS) +_QUERY_VALUE_QUOTE_MAP = _make_quote_map(_QUERY_VALUE_SAFE) +_QUERY_VALUE_DECODE_MAP = _make_decode_map(_QUERY_VALUE_DELIMS) +_FRAGMENT_QUOTE_MAP = _make_quote_map(_FRAGMENT_SAFE) +_FRAGMENT_DECODE_MAP = _make_decode_map(_FRAGMENT_DELIMS) +_UNRESERVED_QUOTE_MAP = _make_quote_map(_UNRESERVED_CHARS) +_UNRESERVED_DECODE_MAP = dict( + [ + (k, v) + for k, v in _HEX_CHAR_MAP.items() + if v.decode("ascii", "replace") in _UNRESERVED_CHARS + ] +) + +_ROOT_PATHS = frozenset(((), (u"",))) + + +def _encode_reserved(text, maximal=True): + # type: (Text, bool) -> Text + """A very comprehensive percent encoding for encoding all + delimiters. Used for arguments to DecodedURL, where a % means a + percent sign, and not the character used by URLs for escaping + bytes. + """ + if maximal: + bytestr = normalize("NFC", text).encode("utf8") + return u"".join([_UNRESERVED_QUOTE_MAP[b] for b in bytestr]) + return u"".join( + [ + _UNRESERVED_QUOTE_MAP[t] if t in _UNRESERVED_CHARS else t + for t in text + ] + ) + + +def _encode_path_part(text, maximal=True): + # type: (Text, bool) -> Text + "Percent-encode a single segment of a URL path." + if maximal: + bytestr = normalize("NFC", text).encode("utf8") + return u"".join([_PATH_PART_QUOTE_MAP[b] for b in bytestr]) + return u"".join( + [_PATH_PART_QUOTE_MAP[t] if t in _PATH_DELIMS else t for t in text] + ) + + +def _encode_schemeless_path_part(text, maximal=True): + # type: (Text, bool) -> Text + """Percent-encode the first segment of a URL path for a URL without a + scheme specified. + """ + if maximal: + bytestr = normalize("NFC", text).encode("utf8") + return u"".join([_SCHEMELESS_PATH_PART_QUOTE_MAP[b] for b in bytestr]) + return u"".join( + [ + _SCHEMELESS_PATH_PART_QUOTE_MAP[t] + if t in _SCHEMELESS_PATH_DELIMS + else t + for t in text + ] + ) + + +def _encode_path_parts( + text_parts, # type: Sequence[Text] + rooted=False, # type: bool + has_scheme=True, # type: bool + has_authority=True, # type: bool + maximal=True, # type: bool +): + # type: (...) -> Sequence[Text] + """ + Percent-encode a tuple of path parts into a complete path. + + Setting *maximal* to False percent-encodes only the reserved + characters that are syntactically necessary for serialization, + preserving any IRI-style textual data. + + Leaving *maximal* set to its default True percent-encodes + everything required to convert a portion of an IRI to a portion of + a URI. + + RFC 3986 3.3: + + If a URI contains an authority component, then the path component + must either be empty or begin with a slash ("/") character. If a URI + does not contain an authority component, then the path cannot begin + with two slash characters ("//"). In addition, a URI reference + (Section 4.1) may be a relative-path reference, in which case the + first path segment cannot contain a colon (":") character. + """ + if not text_parts: + return () + if rooted: + text_parts = (u"",) + tuple(text_parts) + # elif has_authority and text_parts: + # raise Exception('see rfc above') # TODO: too late to fail like this? + encoded_parts = [] # type: List[Text] + if has_scheme: + encoded_parts = [ + _encode_path_part(part, maximal=maximal) if part else part + for part in text_parts + ] + else: + encoded_parts = [_encode_schemeless_path_part(text_parts[0])] + encoded_parts.extend( + [ + _encode_path_part(part, maximal=maximal) if part else part + for part in text_parts[1:] + ] + ) + return tuple(encoded_parts) + + +def _encode_query_key(text, maximal=True): + # type: (Text, bool) -> Text + """ + Percent-encode a single query string key or value. + """ + if maximal: + bytestr = normalize("NFC", text).encode("utf8") + return u"".join([_QUERY_KEY_QUOTE_MAP[b] for b in bytestr]) + return u"".join( + [_QUERY_KEY_QUOTE_MAP[t] if t in _QUERY_KEY_DELIMS else t for t in text] + ) + + +def _encode_query_value(text, maximal=True): + # type: (Text, bool) -> Text + """ + Percent-encode a single query string key or value. + """ + if maximal: + bytestr = normalize("NFC", text).encode("utf8") + return u"".join([_QUERY_VALUE_QUOTE_MAP[b] for b in bytestr]) + return u"".join( + [ + _QUERY_VALUE_QUOTE_MAP[t] if t in _QUERY_VALUE_DELIMS else t + for t in text + ] + ) + + +def _encode_fragment_part(text, maximal=True): + # type: (Text, bool) -> Text + """Quote the fragment part of the URL. Fragments don't have + subdelimiters, so the whole URL fragment can be passed. + """ + if maximal: + bytestr = normalize("NFC", text).encode("utf8") + return u"".join([_FRAGMENT_QUOTE_MAP[b] for b in bytestr]) + return u"".join( + [_FRAGMENT_QUOTE_MAP[t] if t in _FRAGMENT_DELIMS else t for t in text] + ) + + +def _encode_userinfo_part(text, maximal=True): + # type: (Text, bool) -> Text + """Quote special characters in either the username or password + section of the URL. + """ + if maximal: + bytestr = normalize("NFC", text).encode("utf8") + return u"".join([_USERINFO_PART_QUOTE_MAP[b] for b in bytestr]) + return u"".join( + [ + _USERINFO_PART_QUOTE_MAP[t] if t in _USERINFO_DELIMS else t + for t in text + ] + ) + + +# This port list painstakingly curated by hand searching through +# https://www.iana.org/assignments/uri-schemes/uri-schemes.xhtml +# and +# https://www.iana.org/assignments/service-names-port-numbers/service-names-port-numbers.xhtml +SCHEME_PORT_MAP = { + "acap": 674, + "afp": 548, + "dict": 2628, + "dns": 53, + "file": None, + "ftp": 21, + "git": 9418, + "gopher": 70, + "http": 80, + "https": 443, + "imap": 143, + "ipp": 631, + "ipps": 631, + "irc": 194, + "ircs": 6697, + "ldap": 389, + "ldaps": 636, + "mms": 1755, + "msrp": 2855, + "msrps": None, + "mtqp": 1038, + "nfs": 111, + "nntp": 119, + "nntps": 563, + "pop": 110, + "prospero": 1525, + "redis": 6379, + "rsync": 873, + "rtsp": 554, + "rtsps": 322, + "rtspu": 5005, + "sftp": 22, + "smb": 445, + "snmp": 161, + "ssh": 22, + "steam": None, + "svn": 3690, + "telnet": 23, + "ventrilo": 3784, + "vnc": 5900, + "wais": 210, + "ws": 80, + "wss": 443, + "xmpp": None, +} + +# This list of schemes that don't use authorities is also from the link above. +NO_NETLOC_SCHEMES = set( + [ + "urn", + "about", + "bitcoin", + "blob", + "data", + "geo", + "magnet", + "mailto", + "news", + "pkcs11", + "sip", + "sips", + "tel", + ] +) +# As of Mar 11, 2017, there were 44 netloc schemes, and 13 non-netloc + + +def register_scheme(text, uses_netloc=True, default_port=None): + # type: (Text, bool, Optional[int]) -> None + """Registers new scheme information, resulting in correct port and + slash behavior from the URL object. There are dozens of standard + schemes preregistered, so this function is mostly meant for + proprietary internal customizations or stopgaps on missing + standards information. If a scheme seems to be missing, please + `file an issue`_! + + Args: + text (Text): A string representation of the scheme. + (the 'http' in 'http://hatnote.com') + uses_netloc (bool): Does the scheme support specifying a + network host? For instance, "http" does, "mailto" does + not. Defaults to True. + default_port (Optional[int]): The default port, if any, for + netloc-using schemes. + + .. _file an issue: https://github.com/mahmoud/hyperlink/issues + """ + text = text.lower() + if default_port is not None: + try: + default_port = int(default_port) + except (ValueError, TypeError): + raise ValueError( + "default_port expected integer or None, not %r" + % (default_port,) + ) + + if uses_netloc is True: + SCHEME_PORT_MAP[text] = default_port + elif uses_netloc is False: + if default_port is not None: + raise ValueError( + "unexpected default port while specifying" + " non-netloc scheme: %r" % default_port + ) + NO_NETLOC_SCHEMES.add(text) + else: + raise ValueError("uses_netloc expected bool, not: %r" % uses_netloc) + + return + + +def scheme_uses_netloc(scheme, default=None): + # type: (Text, Optional[bool]) -> Optional[bool] + """Whether or not a URL uses :code:`:` or :code:`://` to separate the + scheme from the rest of the URL depends on the scheme's own + standard definition. There is no way to infer this behavior + from other parts of the URL. A scheme either supports network + locations or it does not. + + The URL type's approach to this is to check for explicitly + registered schemes, with common schemes like HTTP + preregistered. This is the same approach taken by + :mod:`urlparse`. + + URL adds two additional heuristics if the scheme as a whole is + not registered. First, it attempts to check the subpart of the + scheme after the last ``+`` character. This adds intuitive + behavior for schemes like ``git+ssh``. Second, if a URL with + an unrecognized scheme is loaded, it will maintain the + separator it sees. + """ + if not scheme: + return False + scheme = scheme.lower() + if scheme in SCHEME_PORT_MAP: + return True + if scheme in NO_NETLOC_SCHEMES: + return False + if scheme.split("+")[-1] in SCHEME_PORT_MAP: + return True + return default + + +class URLParseError(ValueError): + """Exception inheriting from :exc:`ValueError`, raised when failing to + parse a URL. Mostly raised on invalid ports and IPv6 addresses. + """ + + pass + + +def _optional(argument, default): + # type: (Any, Any) -> Any + if argument is _UNSET: + return default + else: + return argument + + +def _typecheck(name, value, *types): + # type: (Text, T, Type[Any]) -> T + """ + Check that the given *value* is one of the given *types*, or raise an + exception describing the problem using *name*. + """ + if not types: + raise ValueError("expected one or more types, maybe use _textcheck?") + if not isinstance(value, types): + raise TypeError( + "expected %s for %s, got %r" + % (" or ".join([t.__name__ for t in types]), name, value) + ) + return value + + +def _textcheck(name, value, delims=frozenset(), nullable=False): + # type: (Text, T, Iterable[Text], bool) -> T + if not isinstance(value, Text): + if nullable and value is None: + # used by query string values + return value # type: ignore[unreachable] + else: + str_name = "unicode" if PY2 else "str" + exp = str_name + " or NoneType" if nullable else str_name + raise TypeError("expected %s for %s, got %r" % (exp, name, value)) + if delims and set(value) & set(delims): # TODO: test caching into regexes + raise ValueError( + "one or more reserved delimiters %s present in %s: %r" + % ("".join(delims), name, value) + ) + return value # type: ignore[return-value] # T vs. Text + + +def iter_pairs(iterable): + # type: (Iterable[Any]) -> Iterator[Any] + """ + Iterate over the (key, value) pairs in ``iterable``. + + This handles dictionaries sensibly, and falls back to assuming the + iterable yields (key, value) pairs. This behaviour is similar to + what Python's ``dict()`` constructor does. + """ + if isinstance(iterable, MappingABC): + iterable = iterable.items() + return iter(iterable) + + +def _decode_unreserved(text, normalize_case=False, encode_stray_percents=False): + # type: (Text, bool, bool) -> Text + return _percent_decode( + text, + normalize_case=normalize_case, + encode_stray_percents=encode_stray_percents, + _decode_map=_UNRESERVED_DECODE_MAP, + ) + + +def _decode_userinfo_part( + text, normalize_case=False, encode_stray_percents=False +): + # type: (Text, bool, bool) -> Text + return _percent_decode( + text, + normalize_case=normalize_case, + encode_stray_percents=encode_stray_percents, + _decode_map=_USERINFO_DECODE_MAP, + ) + + +def _decode_path_part(text, normalize_case=False, encode_stray_percents=False): + # type: (Text, bool, bool) -> Text + """ + >>> _decode_path_part(u'%61%77%2f%7a') + u'aw%2fz' + >>> _decode_path_part(u'%61%77%2f%7a', normalize_case=True) + u'aw%2Fz' + """ + return _percent_decode( + text, + normalize_case=normalize_case, + encode_stray_percents=encode_stray_percents, + _decode_map=_PATH_DECODE_MAP, + ) + + +def _decode_query_key(text, normalize_case=False, encode_stray_percents=False): + # type: (Text, bool, bool) -> Text + return _percent_decode( + text, + normalize_case=normalize_case, + encode_stray_percents=encode_stray_percents, + _decode_map=_QUERY_KEY_DECODE_MAP, + ) + + +def _decode_query_value( + text, normalize_case=False, encode_stray_percents=False +): + # type: (Text, bool, bool) -> Text + return _percent_decode( + text, + normalize_case=normalize_case, + encode_stray_percents=encode_stray_percents, + _decode_map=_QUERY_VALUE_DECODE_MAP, + ) + + +def _decode_fragment_part( + text, normalize_case=False, encode_stray_percents=False +): + # type: (Text, bool, bool) -> Text + return _percent_decode( + text, + normalize_case=normalize_case, + encode_stray_percents=encode_stray_percents, + _decode_map=_FRAGMENT_DECODE_MAP, + ) + + +def _percent_decode( + text, # type: Text + normalize_case=False, # type: bool + subencoding="utf-8", # type: Text + raise_subencoding_exc=False, # type: bool + encode_stray_percents=False, # type: bool + _decode_map=_HEX_CHAR_MAP, # type: Mapping[bytes, bytes] +): + # type: (...) -> Text + """Convert percent-encoded text characters to their normal, + human-readable equivalents. + + All characters in the input text must be encodable by + *subencoding*. All special characters underlying the values in the + percent-encoding must be decodable as *subencoding*. If a + non-*subencoding*-valid string is passed, the original text is + returned with no changes applied. + + Only called by field-tailored variants, e.g., + :func:`_decode_path_part`, as every percent-encodable part of the + URL has characters which should not be percent decoded. + + >>> _percent_decode(u'abc%20def') + u'abc def' + + Args: + text: Text with percent-encoding present. + normalize_case: Whether undecoded percent segments, such as encoded + delimiters, should be uppercased, per RFC 3986 Section 2.1. + See :func:`_decode_path_part` for an example. + subencoding: The name of the encoding underlying the percent-encoding. + raise_subencoding_exc: Whether an error in decoding the bytes + underlying the percent-decoding should be raised. + + Returns: + Text: The percent-decoded version of *text*, decoded by *subencoding*. + """ + try: + quoted_bytes = text.encode(subencoding) + except UnicodeEncodeError: + return text + + bits = quoted_bytes.split(b"%") + if len(bits) == 1: + return text + + res = [bits[0]] + append = res.append + + for item in bits[1:]: + hexpair, rest = item[:2], item[2:] + try: + append(_decode_map[hexpair]) + append(rest) + except KeyError: + pair_is_hex = hexpair in _HEX_CHAR_MAP + if pair_is_hex or not encode_stray_percents: + append(b"%") + else: + # if it's undecodable, treat as a real percent sign, + # which is reserved (because it wasn't in the + # context-aware _decode_map passed in), and should + # stay in an encoded state. + append(b"%25") + if normalize_case and pair_is_hex: + append(hexpair.upper()) + append(rest) + else: + append(item) + + unquoted_bytes = b"".join(res) + + try: + return unquoted_bytes.decode(subencoding) + except UnicodeDecodeError: + if raise_subencoding_exc: + raise + return text + + +def _decode_host(host): + # type: (Text) -> Text + """Decode a host from ASCII-encodable text to IDNA-decoded text. If + the host text is not ASCII, it is returned unchanged, as it is + presumed that it is already IDNA-decoded. + + Some technical details: _decode_host is built on top of the "idna" + package, which has some quirks: + + Capital letters are not valid IDNA2008. The idna package will + raise an exception like this on capital letters: + + > idna.core.InvalidCodepoint: Codepoint U+004B at position 1 ... not allowed + + However, if a segment of a host (i.e., something in + url.host.split('.')) is already ASCII, idna doesn't perform its + usual checks. In fact, for capital letters it automatically + lowercases them. + + This check and some other functionality can be bypassed by passing + uts46=True to idna.encode/decode. This allows a more permissive and + convenient interface. So far it seems like the balanced approach. + + Example output (from idna==2.6): + + >> idna.encode(u'mahmöud.io') + 'xn--mahmud-zxa.io' + >> idna.encode(u'Mahmöud.io') + Traceback (most recent call last): + File "", line 1, in + File "/home/mahmoud/virtualenvs/hyperlink/local/lib/python2.7/site-packages/idna/core.py", line 355, in encode + result.append(alabel(label)) + File "/home/mahmoud/virtualenvs/hyperlink/local/lib/python2.7/site-packages/idna/core.py", line 276, in alabel + check_label(label) + File "/home/mahmoud/virtualenvs/hyperlink/local/lib/python2.7/site-packages/idna/core.py", line 253, in check_label + raise InvalidCodepoint('Codepoint {0} at position {1} of {2} not allowed'.format(_unot(cp_value), pos+1, repr(label))) + idna.core.InvalidCodepoint: Codepoint U+004D at position 1 of u'Mahm\xf6ud' not allowed + >> idna.encode(u'Mahmoud.io') + 'Mahmoud.io' + + # Similar behavior for decodes below + >> idna.decode(u'Mahmoud.io') + u'mahmoud.io + >> idna.decode(u'Méhmoud.io', uts46=True) + u'm\xe9hmoud.io' + """ # noqa: E501 + if not host: + return u"" + try: + host_bytes = host.encode("ascii") + except UnicodeEncodeError: + host_text = host + else: + try: + host_text = idna_decode(host_bytes, uts46=True) + except ValueError: + # only reached on "narrow" (UCS-2) Python builds <3.4, see #7 + # NOTE: not going to raise here, because there's no + # ambiguity in the IDNA, and the host is still + # technically usable + host_text = host + return host_text + + +def _resolve_dot_segments(path): + # type: (Sequence[Text]) -> Sequence[Text] + """Normalize the URL path by resolving segments of '.' and '..'. For + more details, see `RFC 3986 section 5.2.4, Remove Dot Segments`_. + + Args: + path: sequence of path segments in text form + + Returns: + A new sequence of path segments with the '.' and '..' elements removed + and resolved. + + .. _RFC 3986 section 5.2.4, Remove Dot Segments: https://tools.ietf.org/html/rfc3986#section-5.2.4 + """ # noqa: E501 + segs = [] # type: List[Text] + + for seg in path: + if seg == u".": + pass + elif seg == u"..": + if segs: + segs.pop() + else: + segs.append(seg) + + if list(path[-1:]) in ([u"."], [u".."]): + segs.append(u"") + + return segs + + +def parse_host(host): + # type: (Text) -> Tuple[Optional[AddressFamily], Text] + """Parse the host into a tuple of ``(family, host)``, where family + is the appropriate :mod:`socket` module constant when the host is + an IP address. Family is ``None`` when the host is not an IP. + + Will raise :class:`URLParseError` on invalid IPv6 constants. + + Returns: + family (socket constant or None), host (string) + + >>> import socket + >>> parse_host('googlewebsite.com') == (None, 'googlewebsite.com') + True + >>> parse_host('::1') == (socket.AF_INET6, '::1') + True + >>> parse_host('192.168.1.1') == (socket.AF_INET, '192.168.1.1') + True + """ + if not host: + return None, u"" + + if u":" in host: + try: + inet_pton(AF_INET6, host) + except socket.error as se: + raise URLParseError("invalid IPv6 host: %r (%r)" % (host, se)) + except UnicodeEncodeError: + pass # TODO: this can't be a real host right? + else: + family = AF_INET6 # type: Optional[AddressFamily] + else: + try: + inet_pton(AF_INET, host) + except (socket.error, UnicodeEncodeError): + family = None # not an IP + else: + family = AF_INET + + return family, host + + +class URL(object): + r"""From blogs to billboards, URLs are so common, that it's easy to + overlook their complexity and power. With hyperlink's + :class:`URL` type, working with URLs doesn't have to be hard. + + URLs are made of many parts. Most of these parts are officially + named in `RFC 3986`_ and this diagram may prove handy in identifying + them:: + + foo://user:pass@example.com:8042/over/there?name=ferret#nose + \_/ \_______/ \_________/ \__/\_________/ \_________/ \__/ + | | | | | | | + scheme userinfo host port path query fragment + + While :meth:`~URL.from_text` is used for parsing whole URLs, the + :class:`URL` constructor builds a URL from the individual + components, like so:: + + >>> from hyperlink import URL + >>> url = URL(scheme=u'https', host=u'example.com', path=[u'hello', u'world']) + >>> print(url.to_text()) + https://example.com/hello/world + + The constructor runs basic type checks. All strings are expected + to be decoded (:class:`unicode` in Python 2). All arguments are + optional, defaulting to appropriately empty values. A full list of + constructor arguments is below. + + Args: + scheme (Optional[Text]): The text name of the scheme. + host (Optional[Text]): The host portion of the network location + port (Optional[int]): The port part of the network location. If + ``None`` or no port is passed, the port will default to + the default port of the scheme, if it is known. See the + ``SCHEME_PORT_MAP`` and :func:`register_default_port` + for more info. + path (Iterable[Text]): A tuple of strings representing the + slash-separated parts of the path. + query (Sequence[Tuple[Text, Optional[Text]]]): The query parameters, as + a dictionary or as an sequence of key-value pairs. + fragment (Text): The fragment part of the URL. + rooted (bool): A rooted URL is one which indicates an absolute path. + This is True on any URL that includes a host, or any relative URL + that starts with a slash. + userinfo (Text): The username or colon-separated + username:password pair. + uses_netloc (Optional[bool]): Indicates whether ``://`` (the "netloc + separator") will appear to separate the scheme from the *path* in + cases where no host is present. Setting this to ``True`` is a + non-spec-compliant affordance for the common practice of having URIs + that are *not* URLs (cannot have a 'host' part) but nevertheless use + the common ``://`` idiom that most people associate with URLs; + e.g. ``message:`` URIs like ``message://message-id`` being + equivalent to ``message:message-id``. This may be inferred based on + the scheme depending on whether :func:`register_scheme` has been + used to register the scheme and should not be passed directly unless + you know the scheme works like this and you know it has not been + registered. + + All of these parts are also exposed as read-only attributes of + URL instances, along with several useful methods. + + .. _RFC 3986: https://tools.ietf.org/html/rfc3986 + .. _RFC 3987: https://tools.ietf.org/html/rfc3987 + """ # noqa: E501 + + def __init__( + self, + scheme=None, # type: Optional[Text] + host=None, # type: Optional[Text] + path=(), # type: Iterable[Text] + query=(), # type: QueryParameters + fragment=u"", # type: Text + port=None, # type: Optional[int] + rooted=None, # type: Optional[bool] + userinfo=u"", # type: Text + uses_netloc=None, # type: Optional[bool] + ): + # type: (...) -> None + if host is not None and scheme is None: + scheme = u"http" # TODO: why + if port is None and scheme is not None: + port = SCHEME_PORT_MAP.get(scheme) + if host and query and not path: + # per RFC 3986 6.2.3, "a URI that uses the generic syntax + # for authority with an empty path should be normalized to + # a path of '/'." + path = (u"",) + + # Now that we're done detecting whether they were passed, we can set + # them to their defaults: + if scheme is None: + scheme = u"" + if host is None: + host = u"" + if rooted is None: + rooted = bool(host) + + # Set attributes. + self._scheme = _textcheck("scheme", scheme) + if self._scheme: + if not _SCHEME_RE.match(self._scheme): + raise ValueError( + 'invalid scheme: %r. Only alphanumeric, "+",' + ' "-", and "." allowed. Did you meant to call' + " %s.from_text()?" % (self._scheme, self.__class__.__name__) + ) + + _, self._host = parse_host(_textcheck("host", host, "/?#@")) + if isinstance(path, Text): + raise TypeError( + "expected iterable of text for path, not: %r" % (path,) + ) + self._path = tuple( + (_textcheck("path segment", segment, "/?#") for segment in path) + ) + self._query = tuple( + ( + _textcheck("query parameter name", k, "&=#"), + _textcheck("query parameter value", v, "&#", nullable=True), + ) + for k, v in iter_pairs(query) + ) + self._fragment = _textcheck("fragment", fragment) + self._port = _typecheck("port", port, int, NoneType) + self._rooted = _typecheck("rooted", rooted, bool) + self._userinfo = _textcheck("userinfo", userinfo, "/?#@") + + if uses_netloc is None: + uses_netloc = scheme_uses_netloc(self._scheme, uses_netloc) + self._uses_netloc = _typecheck( + "uses_netloc", uses_netloc, bool, NoneType + ) + will_have_authority = self._host or ( + self._port and self._port != SCHEME_PORT_MAP.get(scheme) + ) + if will_have_authority: + # fixup for rooted consistency; if there's any 'authority' + # represented in the textual URL, then the path must be rooted, and + # we're definitely using a netloc (there must be a ://). + self._rooted = True + self._uses_netloc = True + if (not self._rooted) and self.path[:1] == (u"",): + self._rooted = True + self._path = self._path[1:] + if not will_have_authority and self._path and not self._rooted: + # If, after fixing up the path, there *is* a path and it *isn't* + # rooted, then we are definitely not using a netloc; if we did, it + # would make the path (erroneously) look like a hostname. + self._uses_netloc = False + + def get_decoded_url(self, lazy=False): + # type: (bool) -> DecodedURL + try: + return self._decoded_url + except AttributeError: + self._decoded_url = DecodedURL(self, lazy=lazy) # type: DecodedURL + return self._decoded_url + + @property + def scheme(self): + # type: () -> Text + """The scheme is a string, and the first part of an absolute URL, the + part before the first colon, and the part which defines the + semantics of the rest of the URL. Examples include "http", + "https", "ssh", "file", "mailto", and many others. See + :func:`~hyperlink.register_scheme()` for more info. + """ + return self._scheme + + @property + def host(self): + # type: () -> Text + """The host is a string, and the second standard part of an absolute + URL. When present, a valid host must be a domain name, or an + IP (v4 or v6). It occurs before the first slash, or the second + colon, if a :attr:`~hyperlink.URL.port` is provided. + """ + return self._host + + @property + def port(self): + # type: () -> Optional[int] + """The port is an integer that is commonly used in connecting to the + :attr:`host`, and almost never appears without it. + + When not present in the original URL, this attribute defaults + to the scheme's default port. If the scheme's default port is + not known, and the port is not provided, this attribute will + be set to None. + + >>> URL.from_text(u'http://example.com/pa/th').port + 80 + >>> URL.from_text(u'foo://example.com/pa/th').port + >>> URL.from_text(u'foo://example.com:8042/pa/th').port + 8042 + + .. note:: + + Per the standard, when the port is the same as the schemes + default port, it will be omitted in the text URL. + """ + return self._port + + @property + def path(self): + # type: () -> Sequence[Text] + """A tuple of strings, created by splitting the slash-separated + hierarchical path. Started by the first slash after the host, + terminated by a "?", which indicates the start of the + :attr:`~hyperlink.URL.query` string. + """ + return self._path + + @property + def query(self): + # type: () -> QueryPairs + """Tuple of pairs, created by splitting the ampersand-separated + mapping of keys and optional values representing + non-hierarchical data used to identify the resource. Keys are + always strings. Values are strings when present, or None when + missing. + + For more operations on the mapping, see + :meth:`~hyperlink.URL.get()`, :meth:`~hyperlink.URL.add()`, + :meth:`~hyperlink.URL.set()`, and + :meth:`~hyperlink.URL.delete()`. + """ + return self._query + + @property + def fragment(self): + # type: () -> Text + """A string, the last part of the URL, indicated by the first "#" + after the :attr:`~hyperlink.URL.path` or + :attr:`~hyperlink.URL.query`. Enables indirect identification + of a secondary resource, like an anchor within an HTML page. + """ + return self._fragment + + @property + def rooted(self): + # type: () -> bool + """Whether or not the path starts with a forward slash (``/``). + + This is taken from the terminology in the BNF grammar, + specifically the "path-rootless", rule, since "absolute path" + and "absolute URI" are somewhat ambiguous. :attr:`path` does + not contain the implicit prefixed ``"/"`` since that is + somewhat awkward to work with. + """ + return self._rooted + + @property + def userinfo(self): + # type: () -> Text + """The colon-separated string forming the username-password + combination. + """ + return self._userinfo + + @property + def uses_netloc(self): + # type: () -> Optional[bool] + """ + Indicates whether ``://`` (the "netloc separator") will appear to + separate the scheme from the *path* in cases where no host is present. + """ + return self._uses_netloc + + @property + def user(self): + # type: () -> Text + """ + The user portion of :attr:`~hyperlink.URL.userinfo`. + """ + return self.userinfo.split(u":")[0] + + def authority(self, with_password=False, **kw): + # type: (bool, Any) -> Text + """Compute and return the appropriate host/port/userinfo combination. + + >>> url = URL.from_text(u'http://user:pass@localhost:8080/a/b?x=y') + >>> url.authority() + u'user:@localhost:8080' + >>> url.authority(with_password=True) + u'user:pass@localhost:8080' + + Args: + with_password (bool): Whether the return value of this + method include the password in the URL, if it is + set. Defaults to False. + + Returns: + Text: The authority (network location and user information) portion + of the URL. + """ + # first, a bit of twisted compat + with_password = kw.pop("includeSecrets", with_password) + if kw: + raise TypeError("got unexpected keyword arguments: %r" % kw.keys()) + host = self.host + if ":" in host: + hostport = ["[" + host + "]"] + else: + hostport = [self.host] + if self.port != SCHEME_PORT_MAP.get(self.scheme): + hostport.append(Text(self.port)) + authority = [] + if self.userinfo: + userinfo = self.userinfo + if not with_password and u":" in userinfo: + userinfo = userinfo[: userinfo.index(u":") + 1] + authority.append(userinfo) + authority.append(u":".join(hostport)) + return u"@".join(authority) + + def __eq__(self, other): + # type: (Any) -> bool + if not isinstance(other, self.__class__): + return NotImplemented + for attr in [ + "scheme", + "userinfo", + "host", + "query", + "fragment", + "port", + "uses_netloc", + "rooted", + ]: + if getattr(self, attr) != getattr(other, attr): + return False + if self.path == other.path or ( + self.path in _ROOT_PATHS and other.path in _ROOT_PATHS + ): + return True + return False + + def __ne__(self, other): + # type: (Any) -> bool + if not isinstance(other, self.__class__): + return NotImplemented + return not self.__eq__(other) + + def __hash__(self): + # type: () -> int + return hash( + ( + self.__class__, + self.scheme, + self.userinfo, + self.host, + self.path, + self.query, + self.fragment, + self.port, + self.rooted, + self.uses_netloc, + ) + ) + + @property + def absolute(self): + # type: () -> bool + """Whether or not the URL is "absolute". Absolute URLs are complete + enough to resolve to a network resource without being relative + to a base URI. + + >>> URL.from_text(u'http://wikipedia.org/').absolute + True + >>> URL.from_text(u'?a=b&c=d').absolute + False + + Absolute URLs must have both a scheme and a host set. + """ + return bool(self.scheme and self.host) + + def replace( + self, + scheme=_UNSET, # type: Optional[Text] + host=_UNSET, # type: Optional[Text] + path=_UNSET, # type: Iterable[Text] + query=_UNSET, # type: QueryParameters + fragment=_UNSET, # type: Text + port=_UNSET, # type: Optional[int] + rooted=_UNSET, # type: Optional[bool] + userinfo=_UNSET, # type: Text + uses_netloc=_UNSET, # type: Optional[bool] + ): + # type: (...) -> URL + """:class:`URL` objects are immutable, which means that attributes + are designed to be set only once, at construction. Instead of + modifying an existing URL, one simply creates a copy with the + desired changes. + + If any of the following arguments is omitted, it defaults to + the value on the current URL. + + Args: + scheme (Optional[Text]): The text name of the scheme. + host (Optional[Text]): The host portion of the network location. + path (Iterable[Text]): A tuple of strings representing the + slash-separated parts of the path. + query (Sequence[Tuple[Text, Optional[Text]]]): The query + parameters, as a dictionary or as an sequence of key-value + pairs. + fragment (Text): The fragment part of the URL. + port (Optional[int]): The port part of the network location. + rooted (Optional[bool]): Whether or not the path begins with a + slash. + userinfo (Text): The username or colon-separated username:password + pair. + uses_netloc (bool): Indicates whether ``://`` (the "netloc + separator") will appear to separate the scheme from the *path* + in cases where no host is present. Setting this to ``True`` is + a non-spec-compliant affordance for the common practice of + having URIs that are *not* URLs (cannot have a 'host' part) but + nevertheless use the common ``://`` idiom that most people + associate with URLs; e.g. ``message:`` URIs like + ``message://message-id`` being equivalent to + ``message:message-id``. This may be inferred based on the + scheme depending on whether :func:`register_scheme` has been + used to register the scheme and should not be passed directly + unless you know the scheme works like this and you know it has + not been registered. + + Returns: + URL: A copy of the current :class:`URL`, with new values for + parameters passed. + """ + if scheme is not _UNSET and scheme != self.scheme: + # when changing schemes, reset the explicit uses_netloc preference + # to honor the new scheme. + uses_netloc = None + return self.__class__( + scheme=_optional(scheme, self.scheme), + host=_optional(host, self.host), + path=_optional(path, self.path), + query=_optional(query, self.query), + fragment=_optional(fragment, self.fragment), + port=_optional(port, self.port), + rooted=_optional(rooted, self.rooted), + userinfo=_optional(userinfo, self.userinfo), + uses_netloc=_optional(uses_netloc, self.uses_netloc), + ) + + @classmethod + def from_text(cls, text): + # type: (Text) -> URL + """Whereas the :class:`URL` constructor is useful for constructing + URLs from parts, :meth:`~URL.from_text` supports parsing whole + URLs from their string form:: + + >>> URL.from_text(u'http://example.com') + URL.from_text(u'http://example.com') + >>> URL.from_text(u'?a=b&x=y') + URL.from_text(u'?a=b&x=y') + + As you can see above, it's also used as the :func:`repr` of + :class:`URL` objects. The natural counterpart to + :func:`~URL.to_text()`. This method only accepts *text*, so be + sure to decode those bytestrings. + + Args: + text (Text): A valid URL string. + + Returns: + URL: The structured object version of the parsed string. + + .. note:: + + Somewhat unexpectedly, URLs are a far more permissive + format than most would assume. Many strings which don't + look like URLs are still valid URLs. As a result, this + method only raises :class:`URLParseError` on invalid port + and IPv6 values in the host portion of the URL. + """ + um = _URL_RE.match(_textcheck("text", text)) + if um is None: + raise URLParseError("could not parse url: %r" % text) + gs = um.groupdict() + + au_text = gs["authority"] or u"" + au_m = _AUTHORITY_RE.match(au_text) + if au_m is None: + raise URLParseError( + "invalid authority %r in url: %r" % (au_text, text) + ) + au_gs = au_m.groupdict() + if au_gs["bad_host"]: + raise URLParseError( + "invalid host %r in url: %r" % (au_gs["bad_host"], text) + ) + + userinfo = au_gs["userinfo"] or u"" + + host = au_gs["ipv6_host"] or au_gs["plain_host"] + port = au_gs["port"] + if port is not None: + try: + port = int(port) # type: ignore[assignment] # FIXME, see below + except ValueError: + if not port: # TODO: excessive? + raise URLParseError("port must not be empty: %r" % au_text) + raise URLParseError("expected integer for port, not %r" % port) + + scheme = gs["scheme"] or u"" + fragment = gs["fragment"] or u"" + uses_netloc = bool(gs["_netloc_sep"]) + + if gs["path"]: + path = tuple(gs["path"].split(u"/")) + if not path[0]: + path = path[1:] + rooted = True + else: + rooted = False + else: + path = () + rooted = bool(au_text) + if gs["query"]: + query = tuple( + ( + qe.split(u"=", 1) # type: ignore[misc] + if u"=" in qe + else (qe, None) + ) + for qe in gs["query"].split(u"&") + ) # type: QueryPairs + else: + query = () + return cls( + scheme, + host, + path, + query, + fragment, + port, # type: ignore[arg-type] # FIXME, see above + rooted, + userinfo, + uses_netloc, + ) + + def normalize( + self, + scheme=True, + host=True, + path=True, + query=True, + fragment=True, + userinfo=True, + percents=True, + ): + # type: (bool, bool, bool, bool, bool, bool, bool) -> URL + """Return a new URL object with several standard normalizations + applied: + + * Decode unreserved characters (`RFC 3986 2.3`_) + * Uppercase remaining percent-encoded octets (`RFC 3986 2.1`_) + * Convert scheme and host casing to lowercase (`RFC 3986 3.2.2`_) + * Resolve any "." and ".." references in the path (`RFC 3986 6.2.2.3`_) + * Ensure an ending slash on URLs with an empty path (`RFC 3986 6.2.3`_) + * Encode any stray percent signs (`%`) in percent-encoded + fields (path, query, fragment, userinfo) (`RFC 3986 2.4`_) + + All are applied by default, but normalizations can be disabled + per-part by passing `False` for that part's corresponding + name. + + Args: + scheme (bool): Convert the scheme to lowercase + host (bool): Convert the host to lowercase + path (bool): Normalize the path (see above for details) + query (bool): Normalize the query string + fragment (bool): Normalize the fragment + userinfo (bool): Normalize the userinfo + percents (bool): Encode isolated percent signs for any + percent-encoded fields which are being normalized + (defaults to True). + + >>> url = URL.from_text(u'Http://example.COM/a/../b/./c%2f?%61%') + >>> print(url.normalize().to_text()) + http://example.com/b/c%2F?a%25 + + .. _RFC 3986 3.2.2: https://tools.ietf.org/html/rfc3986#section-3.2.2 + .. _RFC 3986 2.3: https://tools.ietf.org/html/rfc3986#section-2.3 + .. _RFC 3986 2.1: https://tools.ietf.org/html/rfc3986#section-2.1 + .. _RFC 3986 6.2.2.3: https://tools.ietf.org/html/rfc3986#section-6.2.2.3 + .. _RFC 3986 6.2.3: https://tools.ietf.org/html/rfc3986#section-6.2.3 + .. _RFC 3986 2.4: https://tools.ietf.org/html/rfc3986#section-2.4 + """ # noqa: E501 + kw = {} # type: Dict[str, Any] + if scheme: + kw["scheme"] = self.scheme.lower() + if host: + kw["host"] = self.host.lower() + + def _dec_unres(target): + # type: (Text) -> Text + return _decode_unreserved( + target, normalize_case=True, encode_stray_percents=percents + ) + + if path: + if self.path: + kw["path"] = [ + _dec_unres(p) for p in _resolve_dot_segments(self.path) + ] + else: + kw["path"] = (u"",) + if query: + kw["query"] = [ + (_dec_unres(k), _dec_unres(v) if v else v) + for k, v in self.query + ] + if fragment: + kw["fragment"] = _dec_unres(self.fragment) + if userinfo: + kw["userinfo"] = u":".join( + [_dec_unres(p) for p in self.userinfo.split(":", 1)] + ) + + return self.replace(**kw) + + def child(self, *segments): + # type: (Text) -> URL + """Make a new :class:`URL` where the given path segments are a child + of this URL, preserving other parts of the URL, including the + query string and fragment. + + For example:: + + >>> url = URL.from_text(u'http://localhost/a/b?x=y') + >>> child_url = url.child(u"c", u"d") + >>> child_url.to_text() + u'http://localhost/a/b/c/d?x=y' + + Args: + segments (Text): Additional parts to be joined and added to + the path, like :func:`os.path.join`. Special characters + in segments will be percent encoded. + + Returns: + URL: A copy of the current URL with the extra path segments. + """ + if not segments: + return self + + segments = [ # type: ignore[assignment] # variable is tuple + _textcheck("path segment", s) for s in segments + ] + new_path = tuple(self.path) + if self.path and self.path[-1] == u"": + new_path = new_path[:-1] + new_path += tuple(_encode_path_parts(segments, maximal=False)) + return self.replace(path=new_path) + + def sibling(self, segment): + # type: (Text) -> URL + """Make a new :class:`URL` with a single path segment that is a + sibling of this URL path. + + Args: + segment (Text): A single path segment. + + Returns: + URL: A copy of the current URL with the last path segment + replaced by *segment*. Special characters such as + ``/?#`` will be percent encoded. + """ + _textcheck("path segment", segment) + new_path = tuple(self.path)[:-1] + (_encode_path_part(segment),) + return self.replace(path=new_path) + + def click(self, href=u""): + # type: (Union[Text, URL]) -> URL + """Resolve the given URL relative to this URL. + + The resulting URI should match what a web browser would + generate if you visited the current URL and clicked on *href*. + + >>> url = URL.from_text(u'http://blog.hatnote.com/') + >>> url.click(u'/post/155074058790').to_text() + u'http://blog.hatnote.com/post/155074058790' + >>> url = URL.from_text(u'http://localhost/a/b/c/') + >>> url.click(u'../d/./e').to_text() + u'http://localhost/a/b/d/e' + + Args (Text): + href: A string representing a clicked URL. + + Return: + A copy of the current URL with navigation logic applied. + + For more information, see `RFC 3986 section 5`_. + + .. _RFC 3986 section 5: https://tools.ietf.org/html/rfc3986#section-5 + """ + if href: + if isinstance(href, URL): + clicked = href + else: + # TODO: This error message is not completely accurate, + # as URL objects are now also valid, but Twisted's + # test suite (wrongly) relies on this exact message. + _textcheck("relative URL", href) + clicked = URL.from_text(href) + if clicked.absolute: + return clicked + else: + clicked = self + + query = clicked.query + if clicked.scheme and not clicked.rooted: + # Schemes with relative paths are not well-defined. RFC 3986 calls + # them a "loophole in prior specifications" that should be avoided, + # or supported only for backwards compatibility. + raise NotImplementedError( + "absolute URI with rootless path: %r" % (href,) + ) + else: + if clicked.rooted: + path = clicked.path + elif clicked.path: + path = tuple(self.path)[:-1] + tuple(clicked.path) + else: + path = self.path + if not query: + query = self.query + return self.replace( + scheme=clicked.scheme or self.scheme, + host=clicked.host or self.host, + port=clicked.port or self.port, + path=_resolve_dot_segments(path), + query=query, + fragment=clicked.fragment, + ) + + def to_uri(self): + # type: () -> URL + u"""Make a new :class:`URL` instance with all non-ASCII characters + appropriately percent-encoded. This is useful to do in preparation + for sending a :class:`URL` over a network protocol. + + For example:: + + >>> URL.from_text(u'https://ايران.com/foo⇧bar/').to_uri() + URL.from_text(u'https://xn--mgba3a4fra.com/foo%E2%87%A7bar/') + + Returns: + URL: A new instance with its path segments, query parameters, and + hostname encoded, so that they are all in the standard + US-ASCII range. + """ + new_userinfo = u":".join( + [_encode_userinfo_part(p) for p in self.userinfo.split(":", 1)] + ) + new_path = _encode_path_parts( + self.path, has_scheme=bool(self.scheme), rooted=False, maximal=True + ) + new_host = ( + self.host + if not self.host + else idna_encode(self.host, uts46=True).decode("ascii") + ) + return self.replace( + userinfo=new_userinfo, + host=new_host, + path=new_path, + query=tuple( + [ + ( + _encode_query_key(k, maximal=True), + _encode_query_value(v, maximal=True) + if v is not None + else None, + ) + for k, v in self.query + ] + ), + fragment=_encode_fragment_part(self.fragment, maximal=True), + ) + + def to_iri(self): + # type: () -> URL + u"""Make a new :class:`URL` instance with all but a few reserved + characters decoded into human-readable format. + + Percent-encoded Unicode and IDNA-encoded hostnames are + decoded, like so:: + + >>> url = URL.from_text(u'https://xn--mgba3a4fra.example.com/foo%E2%87%A7bar/') + >>> print(url.to_iri().to_text()) + https://ايران.example.com/foo⇧bar/ + + .. note:: + + As a general Python issue, "narrow" (UCS-2) builds of + Python may not be able to fully decode certain URLs, and + the in those cases, this method will return a best-effort, + partially-decoded, URL which is still valid. This issue + does not affect any Python builds 3.4+. + + Returns: + URL: A new instance with its path segments, query parameters, and + hostname decoded for display purposes. + """ # noqa: E501 + new_userinfo = u":".join( + [_decode_userinfo_part(p) for p in self.userinfo.split(":", 1)] + ) + host_text = _decode_host(self.host) + + return self.replace( + userinfo=new_userinfo, + host=host_text, + path=[_decode_path_part(segment) for segment in self.path], + query=tuple( + ( + _decode_query_key(k), + _decode_query_value(v) if v is not None else None, + ) + for k, v in self.query + ), + fragment=_decode_fragment_part(self.fragment), + ) + + def to_text(self, with_password=False): + # type: (bool) -> Text + """Render this URL to its textual representation. + + By default, the URL text will *not* include a password, if one + is set. RFC 3986 considers using URLs to represent such + sensitive information as deprecated. Quoting from RFC 3986, + `section 3.2.1`: + + "Applications should not render as clear text any data after the + first colon (":") character found within a userinfo subcomponent + unless the data after the colon is the empty string (indicating no + password)." + + Args (bool): + with_password: Whether or not to include the password in the URL + text. Defaults to False. + + Returns: + Text: The serialized textual representation of this URL, such as + ``u"http://example.com/some/path?some=query"``. + + The natural counterpart to :class:`URL.from_text()`. + + .. _section 3.2.1: https://tools.ietf.org/html/rfc3986#section-3.2.1 + """ + scheme = self.scheme + authority = self.authority(with_password) + path = "/".join( + _encode_path_parts( + self.path, + rooted=self.rooted, + has_scheme=bool(scheme), + has_authority=bool(authority), + maximal=False, + ) + ) + query_parts = [] + for k, v in self.query: + if v is None: + query_parts.append(_encode_query_key(k, maximal=False)) + else: + query_parts.append( + u"=".join( + ( + _encode_query_key(k, maximal=False), + _encode_query_value(v, maximal=False), + ) + ) + ) + query_string = u"&".join(query_parts) + + fragment = self.fragment + + parts = [] # type: List[Text] + _add = parts.append + if scheme: + _add(scheme) + _add(":") + if authority: + _add("//") + _add(authority) + elif scheme and path[:2] != "//" and self.uses_netloc: + _add("//") + if path: + if scheme and authority and path[:1] != "/": + _add("/") # relpaths with abs authorities auto get '/' + _add(path) + if query_string: + _add("?") + _add(query_string) + if fragment: + _add("#") + _add(fragment) + return u"".join(parts) + + def __repr__(self): + # type: () -> str + """Convert this URL to an representation that shows all of its + constituent parts, as well as being a valid argument to + :func:`eval`. + """ + return "%s.from_text(%r)" % (self.__class__.__name__, self.to_text()) + + def _to_bytes(self): + # type: () -> bytes + """ + Allows for direct usage of URL objects with libraries like + requests, which automatically stringify URL parameters. See + issue #49. + """ + return self.to_uri().to_text().encode("ascii") + + if PY2: + __str__ = _to_bytes + __unicode__ = to_text + else: + __bytes__ = _to_bytes + __str__ = to_text + + # # Begin Twisted Compat Code + asURI = to_uri + asIRI = to_iri + + @classmethod + def fromText(cls, s): + # type: (Text) -> URL + return cls.from_text(s) + + def asText(self, includeSecrets=False): + # type: (bool) -> Text + return self.to_text(with_password=includeSecrets) + + def __dir__(self): + # type: () -> Sequence[Text] + try: + ret = object.__dir__(self) + except AttributeError: + # object.__dir__ == AttributeError # pdw for py2 + ret = dir(self.__class__) + list(self.__dict__.keys()) + ret = sorted(set(ret) - set(["fromText", "asURI", "asIRI", "asText"])) + return ret + + # # End Twisted Compat Code + + def add(self, name, value=None): + # type: (Text, Optional[Text]) -> URL + """Make a new :class:`URL` instance with a given query argument, + *name*, added to it with the value *value*, like so:: + + >>> URL.from_text(u'https://example.com/?x=y').add(u'x') + URL.from_text(u'https://example.com/?x=y&x') + >>> URL.from_text(u'https://example.com/?x=y').add(u'x', u'z') + URL.from_text(u'https://example.com/?x=y&x=z') + + Args: + name (Text): The name of the query parameter to add. + The part before the ``=``. + value (Optional[Text]): The value of the query parameter to add. + The part after the ``=``. Defaults to ``None``, meaning no + value. + + Returns: + URL: A new :class:`URL` instance with the parameter added. + """ + return self.replace(query=self.query + ((name, value),)) + + def set(self, name, value=None): + # type: (Text, Optional[Text]) -> URL + """Make a new :class:`URL` instance with the query parameter *name* + set to *value*. All existing occurences, if any are replaced + by the single name-value pair. + + >>> URL.from_text(u'https://example.com/?x=y').set(u'x') + URL.from_text(u'https://example.com/?x') + >>> URL.from_text(u'https://example.com/?x=y').set(u'x', u'z') + URL.from_text(u'https://example.com/?x=z') + + Args: + name (Text): The name of the query parameter to set. + The part before the ``=``. + value (Optional[Text]): The value of the query parameter to set. + The part after the ``=``. Defaults to ``None``, meaning no + value. + + Returns: + URL: A new :class:`URL` instance with the parameter set. + """ + # Preserve the original position of the query key in the list + q = [(k, v) for (k, v) in self.query if k != name] + idx = next( + (i for (i, (k, v)) in enumerate(self.query) if k == name), -1 + ) + q[idx:idx] = [(name, value)] + return self.replace(query=q) + + def get(self, name): + # type: (Text) -> List[Optional[Text]] + """Get a list of values for the given query parameter, *name*:: + + >>> url = URL.from_text(u'?x=1&x=2') + >>> url.get('x') + [u'1', u'2'] + >>> url.get('y') + [] + + If the given *name* is not set, an empty list is returned. A + list is always returned, and this method raises no exceptions. + + Args: + name (Text): The name of the query parameter to get. + + Returns: + List[Optional[Text]]: A list of all the values associated with the + key, in string form. + """ + return [value for (key, value) in self.query if name == key] + + def remove( + self, + name, # type: Text + value=_UNSET, # type: Text + limit=None, # type: Optional[int] + ): + # type: (...) -> URL + """Make a new :class:`URL` instance with occurrences of the query + parameter *name* removed, or, if *value* is set, parameters + matching *name* and *value*. No exception is raised if the + parameter is not already set. + + Args: + name (Text): The name of the query parameter to remove. + value (Text): Optional value to additionally filter on. + Setting this removes query parameters which match both name + and value. + limit (Optional[int]): Optional maximum number of parameters to + remove. + + Returns: + URL: A new :class:`URL` instance with the parameter removed. + """ + if limit is None: + if value is _UNSET: + nq = [(k, v) for (k, v) in self.query if k != name] + else: + nq = [ + (k, v) + for (k, v) in self.query + if not (k == name and v == value) + ] + else: + nq, removed_count = [], 0 + + for k, v in self.query: + if ( + k == name + and (value is _UNSET or v == value) + and removed_count < limit + ): + removed_count += 1 # drop it + else: + nq.append((k, v)) # keep it + + return self.replace(query=nq) + + +EncodedURL = URL # An alias better describing what the URL really is + +_EMPTY_URL = URL() + + +class DecodedURL(object): + """ + :class:`DecodedURL` is a type designed to act as a higher-level + interface to :class:`URL` and the recommended type for most + operations. By analogy, :class:`DecodedURL` is the + :class:`unicode` to URL's :class:`bytes`. + + :class:`DecodedURL` automatically handles encoding and decoding + all its components, such that all inputs and outputs are in a + maximally-decoded state. Note that this means, for some special + cases, a URL may not "roundtrip" character-for-character, but this + is considered a good tradeoff for the safety of automatic + encoding. + + Otherwise, :class:`DecodedURL` has almost exactly the same API as + :class:`URL`. + + Where applicable, a UTF-8 encoding is presumed. Be advised that + some interactions can raise :exc:`UnicodeEncodeErrors` and + :exc:`UnicodeDecodeErrors`, just like when working with + bytestrings. Examples of such interactions include handling query + strings encoding binary data, and paths containing segments with + special characters encoded with codecs other than UTF-8. + + Args: + url (URL): A :class:`URL` object to wrap. + lazy (bool): Set to True to avoid pre-decode all parts of the URL to + check for validity. Defaults to False. + + .. note:: + + The :class:`DecodedURL` initializer takes a :class:`URL` object, + not URL components, like :class:`URL`. To programmatically + construct a :class:`DecodedURL`, you can use this pattern: + + >>> print(DecodedURL().replace(scheme=u'https', + ... host=u'pypi.org', path=(u'projects', u'hyperlink')).to_text()) + https://pypi.org/projects/hyperlink + + .. versionadded:: 18.0.0 + """ + + def __init__(self, url=_EMPTY_URL, lazy=False): + # type: (URL, bool) -> None + self._url = url + if not lazy: + # cache the following, while triggering any decoding + # issues with decodable fields + self.host, self.userinfo, self.path, self.query, self.fragment + return + + @classmethod + def from_text(cls, text, lazy=False): + # type: (Text, bool) -> DecodedURL + """\ + Make a `DecodedURL` instance from any text string containing a URL. + + Args: + text (Text): Text containing the URL + lazy (bool): Whether to pre-decode all parts of the URL to check for + validity. Defaults to True. + """ + _url = URL.from_text(text) + return cls(_url, lazy=lazy) + + @property + def encoded_url(self): + # type: () -> URL + """Access the underlying :class:`URL` object, which has any special + characters encoded. + """ + return self._url + + def to_text(self, with_password=False): + # type: (bool) -> Text + "Passthrough to :meth:`~hyperlink.URL.to_text()`" + return self._url.to_text(with_password) + + def to_uri(self): + # type: () -> URL + "Passthrough to :meth:`~hyperlink.URL.to_uri()`" + return self._url.to_uri() + + def to_iri(self): + # type: () -> URL + "Passthrough to :meth:`~hyperlink.URL.to_iri()`" + return self._url.to_iri() + + def click(self, href=u""): + # type: (Union[Text, URL, DecodedURL]) -> DecodedURL + """Return a new DecodedURL wrapping the result of + :meth:`~hyperlink.URL.click()` + """ + if isinstance(href, DecodedURL): + href = href._url + return self.__class__(self._url.click(href=href)) + + def sibling(self, segment): + # type: (Text) -> DecodedURL + """Automatically encode any reserved characters in *segment* and + return a new `DecodedURL` wrapping the result of + :meth:`~hyperlink.URL.sibling()` + """ + return self.__class__(self._url.sibling(_encode_reserved(segment))) + + def child(self, *segments): + # type: (Text) -> DecodedURL + """Automatically encode any reserved characters in *segments* and + return a new `DecodedURL` wrapping the result of + :meth:`~hyperlink.URL.child()`. + """ + if not segments: + return self + new_segs = [_encode_reserved(s) for s in segments] + return self.__class__(self._url.child(*new_segs)) + + def normalize( + self, + scheme=True, + host=True, + path=True, + query=True, + fragment=True, + userinfo=True, + percents=True, + ): + # type: (bool, bool, bool, bool, bool, bool, bool) -> DecodedURL + """Return a new `DecodedURL` wrapping the result of + :meth:`~hyperlink.URL.normalize()` + """ + return self.__class__( + self._url.normalize( + scheme, host, path, query, fragment, userinfo, percents + ) + ) + + @property + def absolute(self): + # type: () -> bool + return self._url.absolute + + @property + def scheme(self): + # type: () -> Text + return self._url.scheme + + @property + def host(self): + # type: () -> Text + return _decode_host(self._url.host) + + @property + def port(self): + # type: () -> Optional[int] + return self._url.port + + @property + def rooted(self): + # type: () -> bool + return self._url.rooted + + @property + def path(self): + # type: () -> Sequence[Text] + if not hasattr(self, "_path"): + self._path = tuple( + [ + _percent_decode(p, raise_subencoding_exc=True) + for p in self._url.path + ] + ) + return self._path + + @property + def query(self): + # type: () -> QueryPairs + if not hasattr(self, "_query"): + self._query = cast( + QueryPairs, + tuple( + tuple( + _percent_decode(x, raise_subencoding_exc=True) + if x is not None + else None + for x in (k, v) + ) + for k, v in self._url.query + ), + ) + return self._query + + @property + def fragment(self): + # type: () -> Text + if not hasattr(self, "_fragment"): + frag = self._url.fragment + self._fragment = _percent_decode(frag, raise_subencoding_exc=True) + return self._fragment + + @property + def userinfo(self): + # type: () -> Union[Tuple[str], Tuple[str, str]] + if not hasattr(self, "_userinfo"): + self._userinfo = cast( + Union[Tuple[str], Tuple[str, str]], + tuple( + tuple( + _percent_decode(p, raise_subencoding_exc=True) + for p in self._url.userinfo.split(":", 1) + ) + ), + ) + return self._userinfo + + @property + def user(self): + # type: () -> Text + return self.userinfo[0] + + @property + def uses_netloc(self): + # type: () -> Optional[bool] + return self._url.uses_netloc + + def replace( + self, + scheme=_UNSET, # type: Optional[Text] + host=_UNSET, # type: Optional[Text] + path=_UNSET, # type: Iterable[Text] + query=_UNSET, # type: QueryParameters + fragment=_UNSET, # type: Text + port=_UNSET, # type: Optional[int] + rooted=_UNSET, # type: Optional[bool] + userinfo=_UNSET, # type: Union[Tuple[str], Tuple[str, str]] + uses_netloc=_UNSET, # type: Optional[bool] + ): + # type: (...) -> DecodedURL + """While the signature is the same, this `replace()` differs a little + from URL.replace. For instance, it accepts userinfo as a + tuple, not as a string, handling the case of having a username + containing a `:`. As with the rest of the methods on + DecodedURL, if you pass a reserved character, it will be + automatically encoded instead of an error being raised. + """ + if path is not _UNSET: + path = tuple(_encode_reserved(p) for p in path) + if query is not _UNSET: + query = cast( + QueryPairs, + tuple( + tuple( + _encode_reserved(x) if x is not None else None + for x in (k, v) + ) + for k, v in iter_pairs(query) + ), + ) + if userinfo is not _UNSET: + if len(userinfo) > 2: + raise ValueError( + 'userinfo expected sequence of ["user"] or' + ' ["user", "password"], got %r' % (userinfo,) + ) + userinfo_text = u":".join([_encode_reserved(p) for p in userinfo]) + else: + userinfo_text = _UNSET + new_url = self._url.replace( + scheme=scheme, + host=host, + path=path, + query=query, + fragment=fragment, + port=port, + rooted=rooted, + userinfo=userinfo_text, + uses_netloc=uses_netloc, + ) + return self.__class__(url=new_url) + + def get(self, name): + # type: (Text) -> List[Optional[Text]] + "Get the value of all query parameters whose name matches *name*" + return [v for (k, v) in self.query if name == k] + + def add(self, name, value=None): + # type: (Text, Optional[Text]) -> DecodedURL + """Return a new DecodedURL with the query parameter *name* and *value* + added.""" + return self.replace(query=self.query + ((name, value),)) + + def set(self, name, value=None): + # type: (Text, Optional[Text]) -> DecodedURL + "Return a new DecodedURL with query parameter *name* set to *value*" + query = self.query + q = [(k, v) for (k, v) in query if k != name] + idx = next((i for (i, (k, v)) in enumerate(query) if k == name), -1) + q[idx:idx] = [(name, value)] + return self.replace(query=q) + + def remove( + self, + name, # type: Text + value=_UNSET, # type: Text + limit=None, # type: Optional[int] + ): + # type: (...) -> DecodedURL + """Return a new DecodedURL with query parameter *name* removed. + + Optionally also filter for *value*, as well as cap the number + of parameters removed with *limit*. + """ + if limit is None: + if value is _UNSET: + nq = [(k, v) for (k, v) in self.query if k != name] + else: + nq = [ + (k, v) + for (k, v) in self.query + if not (k == name and v == value) + ] + else: + nq, removed_count = [], 0 + for k, v in self.query: + if ( + k == name + and (value is _UNSET or v == value) + and removed_count < limit + ): + removed_count += 1 # drop it + else: + nq.append((k, v)) # keep it + + return self.replace(query=nq) + + def __repr__(self): + # type: () -> str + cn = self.__class__.__name__ + return "%s(url=%r)" % (cn, self._url) + + def __str__(self): + # type: () -> str + # TODO: the underlying URL's __str__ needs to change to make + # this work as the URL, see #55 + return str(self._url) + + def __eq__(self, other): + # type: (Any) -> bool + if not isinstance(other, self.__class__): + return NotImplemented + return self.normalize().to_uri() == other.normalize().to_uri() + + def __ne__(self, other): + # type: (Any) -> bool + if not isinstance(other, self.__class__): + return NotImplemented + return not self.__eq__(other) + + def __hash__(self): + # type: () -> int + return hash( + ( + self.__class__, + self.scheme, + self.userinfo, + self.host, + self.path, + self.query, + self.fragment, + self.port, + self.rooted, + self.uses_netloc, + ) + ) + + # # Begin Twisted Compat Code + asURI = to_uri + asIRI = to_iri + + @classmethod + def fromText(cls, s, lazy=False): + # type: (Text, bool) -> DecodedURL + return cls.from_text(s, lazy=lazy) + + def asText(self, includeSecrets=False): + # type: (bool) -> Text + return self.to_text(with_password=includeSecrets) + + def __dir__(self): + # type: () -> Sequence[Text] + try: + ret = object.__dir__(self) + except AttributeError: + # object.__dir__ == AttributeError # pdw for py2 + ret = dir(self.__class__) + list(self.__dict__.keys()) + ret = sorted(set(ret) - set(["fromText", "asURI", "asIRI", "asText"])) + return ret + + # # End Twisted Compat Code + + +def parse(url, decoded=True, lazy=False): + # type: (Text, bool, bool) -> Union[URL, DecodedURL] + """ + Automatically turn text into a structured URL object. + + >>> url = parse(u"https://github.com/python-hyper/hyperlink") + >>> print(url.to_text()) + https://github.com/python-hyper/hyperlink + + Args: + url (str): A text string representation of a URL. + + decoded (bool): Whether or not to return a :class:`DecodedURL`, + which automatically handles all + encoding/decoding/quoting/unquoting for all the various + accessors of parts of the URL, or a :class:`URL`, + which has the same API, but requires handling of special + characters for different parts of the URL. + + lazy (bool): In the case of `decoded=True`, this controls + whether the URL is decoded immediately or as accessed. The + default, `lazy=False`, checks all encoded parts of the URL + for decodability. + + .. versionadded:: 18.0.0 + """ + enc_url = EncodedURL.from_text(url) + if not decoded: + return enc_url + dec_url = DecodedURL(enc_url, lazy=lazy) + return dec_url diff --git a/WebCrawler/venv/Lib/site-packages/hyperlink/hypothesis.py b/WebCrawler/venv/Lib/site-packages/hyperlink/hypothesis.py new file mode 100644 index 0000000..6228ad3 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/hyperlink/hypothesis.py @@ -0,0 +1,320 @@ +# -*- coding: utf-8 -*- +""" +Hypothesis strategies. +""" +from __future__ import absolute_import + +try: + import hypothesis + + del hypothesis +except ImportError: + from typing import Tuple + + __all__ = () # type: Tuple[str, ...] +else: + from csv import reader as csv_reader + from os.path import dirname, join + from string import ascii_letters, digits + from sys import maxunicode + from typing import ( + Callable, + Iterable, + List, + Optional, + Sequence, + Text, + TypeVar, + cast, + ) + from gzip import open as open_gzip + + from . import DecodedURL, EncodedURL + + from hypothesis import assume + from hypothesis.strategies import ( + composite, + integers, + lists, + sampled_from, + text, + ) + + from idna import IDNAError, check_label, encode as idna_encode + + __all__ = ( + "decoded_urls", + "encoded_urls", + "hostname_labels", + "hostnames", + "idna_text", + "paths", + "port_numbers", + ) + + T = TypeVar("T") + DrawCallable = Callable[[Callable[..., T]], T] + + try: + unichr + except NameError: # Py3 + unichr = chr # type: Callable[[int], Text] + + def idna_characters(): + # type: () -> Text + """ + Returns a string containing IDNA characters. + """ + global _idnaCharacters + + if not _idnaCharacters: + result = [] + + # Data source "IDNA Derived Properties": + # https://www.iana.org/assignments/idna-tables-6.3.0/ + # idna-tables-6.3.0.xhtml#idna-tables-properties + dataFileName = join( + dirname(__file__), "idna-tables-properties.csv.gz" + ) + with open_gzip(dataFileName) as dataFile: + reader = csv_reader( + (line.decode("utf-8") for line in dataFile), delimiter=",", + ) + next(reader) # Skip header row + for row in reader: + codes, prop, description = row + + if prop != "PVALID": + # CONTEXTO or CONTEXTJ are also allowed, but they come + # with rules, so we're punting on those here. + # See: https://tools.ietf.org/html/rfc5892 + continue + + startEnd = row[0].split("-", 1) + if len(startEnd) == 1: + # No end of range given; use start + startEnd.append(startEnd[0]) + start, end = (int(i, 16) for i in startEnd) + + for i in range(start, end + 1): + if i > maxunicode: # Happens using Py2 on Windows + break + result.append(unichr(i)) + + _idnaCharacters = u"".join(result) + + return _idnaCharacters + + _idnaCharacters = "" # type: Text + + @composite + def idna_text(draw, min_size=1, max_size=None): + # type: (DrawCallable, int, Optional[int]) -> Text + """ + A strategy which generates IDNA-encodable text. + + @param min_size: The minimum number of characters in the text. + C{None} is treated as C{0}. + + @param max_size: The maximum number of characters in the text. + Use C{None} for an unbounded size. + """ + alphabet = idna_characters() + + assert min_size >= 1 + + if max_size is not None: + assert max_size >= 1 + + result = cast( + Text, + draw(text(min_size=min_size, max_size=max_size, alphabet=alphabet)), + ) + + # FIXME: There should be a more efficient way to ensure we produce + # valid IDNA text. + try: + idna_encode(result) + except IDNAError: + assume(False) + + return result + + @composite + def port_numbers(draw, allow_zero=False): + # type: (DrawCallable, bool) -> int + """ + A strategy which generates port numbers. + + @param allow_zero: Whether to allow port C{0} as a possible value. + """ + if allow_zero: + min_value = 0 + else: + min_value = 1 + + return cast(int, draw(integers(min_value=min_value, max_value=65535))) + + @composite + def hostname_labels(draw, allow_idn=True): + # type: (DrawCallable, bool) -> Text + """ + A strategy which generates host name labels. + + @param allow_idn: Whether to allow non-ASCII characters as allowed by + internationalized domain names (IDNs). + """ + if allow_idn: + label = cast(Text, draw(idna_text(min_size=1, max_size=63))) + + try: + label.encode("ascii") + except UnicodeEncodeError: + # If the label doesn't encode to ASCII, then we need to check + # the length of the label after encoding to punycode and adding + # the xn-- prefix. + while len(label.encode("punycode")) > 63 - len("xn--"): + # Rather than bombing out, just trim from the end until it + # is short enough, so hypothesis doesn't have to generate + # new data. + label = label[:-1] + + else: + label = cast( + Text, + draw( + text( + min_size=1, + max_size=63, + alphabet=Text(ascii_letters + digits + u"-"), + ) + ), + ) + + # Filter invalid labels. + # It would be better to reliably avoid generation of bogus labels in + # the first place, but it's hard... + try: + check_label(label) + except UnicodeError: # pragma: no cover (not always drawn) + assume(False) + + return label + + @composite + def hostnames(draw, allow_leading_digit=True, allow_idn=True): + # type: (DrawCallable, bool, bool) -> Text + """ + A strategy which generates host names. + + @param allow_leading_digit: Whether to allow a leading digit in host + names; they were not allowed prior to RFC 1123. + + @param allow_idn: Whether to allow non-ASCII characters as allowed by + internationalized domain names (IDNs). + """ + # Draw first label, filtering out labels with leading digits if needed + labels = [ + cast( + Text, + draw( + hostname_labels(allow_idn=allow_idn).filter( + lambda l: ( + True if allow_leading_digit else l[0] not in digits + ) + ) + ), + ) + ] + # Draw remaining labels + labels += cast( + List[Text], + draw( + lists( + hostname_labels(allow_idn=allow_idn), + min_size=1, + max_size=4, + ) + ), + ) + + # Trim off labels until the total host name length fits in 252 + # characters. This avoids having to filter the data. + while sum(len(label) for label in labels) + len(labels) - 1 > 252: + labels = labels[:-1] + + return u".".join(labels) + + def path_characters(): + # type: () -> str + """ + Returns a string containing valid URL path characters. + """ + global _path_characters + + if _path_characters is None: + + def chars(): + # type: () -> Iterable[Text] + for i in range(maxunicode): + c = unichr(i) + + # Exclude reserved characters + if c in "#/?": + continue + + # Exclude anything not UTF-8 compatible + try: + c.encode("utf-8") + except UnicodeEncodeError: + continue + + yield c + + _path_characters = "".join(chars()) + + return _path_characters + + _path_characters = None # type: Optional[str] + + @composite + def paths(draw): + # type: (DrawCallable) -> Sequence[Text] + return cast( + List[Text], + draw( + lists(text(min_size=1, alphabet=path_characters()), max_size=10) + ), + ) + + @composite + def encoded_urls(draw): + # type: (DrawCallable) -> EncodedURL + """ + A strategy which generates L{EncodedURL}s. + Call the L{EncodedURL.to_uri} method on each URL to get an HTTP + protocol-friendly URI. + """ + port = cast(Optional[int], draw(port_numbers(allow_zero=True))) + host = cast(Text, draw(hostnames())) + path = cast(Sequence[Text], draw(paths())) + + if port == 0: + port = None + + return EncodedURL( + scheme=cast(Text, draw(sampled_from((u"http", u"https")))), + host=host, + port=port, + path=path, + ) + + @composite + def decoded_urls(draw): + # type: (DrawCallable) -> DecodedURL + """ + A strategy which generates L{DecodedURL}s. + Call the L{EncodedURL.to_uri} method on each URL to get an HTTP + protocol-friendly URI. + """ + return DecodedURL(draw(encoded_urls())) diff --git a/WebCrawler/venv/Lib/site-packages/hyperlink/py.typed b/WebCrawler/venv/Lib/site-packages/hyperlink/py.typed new file mode 100644 index 0000000..d2dfd5e --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/hyperlink/py.typed @@ -0,0 +1 @@ +# See: https://www.python.org/dev/peps/pep-0561/ diff --git a/WebCrawler/venv/Lib/site-packages/hyperlink/test/__init__.py b/WebCrawler/venv/Lib/site-packages/hyperlink/test/__init__.py new file mode 100644 index 0000000..e10ca70 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/hyperlink/test/__init__.py @@ -0,0 +1,31 @@ +# -*- coding: utf-8 -*- +""" +Tests for hyperlink +""" + +__all = () + + +def _init_hypothesis(): + # type: () -> None + from os import environ + + if "CI" in environ: + try: + from hypothesis import HealthCheck, settings + except ImportError: + return + + settings.register_profile( + "patience", + settings( + suppress_health_check=[ + HealthCheck.too_slow, + HealthCheck.filter_too_much, + ] + ), + ) + settings.load_profile("patience") + + +_init_hypothesis() diff --git a/WebCrawler/venv/Lib/site-packages/hyperlink/test/__pycache__/__init__.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/hyperlink/test/__pycache__/__init__.cpython-39.pyc new file mode 100644 index 0000000..65ed084 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/hyperlink/test/__pycache__/__init__.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/hyperlink/test/__pycache__/common.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/hyperlink/test/__pycache__/common.cpython-39.pyc new file mode 100644 index 0000000..3f46fa5 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/hyperlink/test/__pycache__/common.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/hyperlink/test/__pycache__/test_common.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/hyperlink/test/__pycache__/test_common.cpython-39.pyc new file mode 100644 index 0000000..92a9636 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/hyperlink/test/__pycache__/test_common.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/hyperlink/test/__pycache__/test_decoded_url.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/hyperlink/test/__pycache__/test_decoded_url.cpython-39.pyc new file mode 100644 index 0000000..2977e9d Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/hyperlink/test/__pycache__/test_decoded_url.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/hyperlink/test/__pycache__/test_hypothesis.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/hyperlink/test/__pycache__/test_hypothesis.cpython-39.pyc new file mode 100644 index 0000000..0c7d8ec Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/hyperlink/test/__pycache__/test_hypothesis.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/hyperlink/test/__pycache__/test_parse.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/hyperlink/test/__pycache__/test_parse.cpython-39.pyc new file mode 100644 index 0000000..22da8f1 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/hyperlink/test/__pycache__/test_parse.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/hyperlink/test/__pycache__/test_scheme_registration.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/hyperlink/test/__pycache__/test_scheme_registration.cpython-39.pyc new file mode 100644 index 0000000..b6bf2ee Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/hyperlink/test/__pycache__/test_scheme_registration.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/hyperlink/test/__pycache__/test_socket.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/hyperlink/test/__pycache__/test_socket.cpython-39.pyc new file mode 100644 index 0000000..e19c1b7 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/hyperlink/test/__pycache__/test_socket.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/hyperlink/test/__pycache__/test_url.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/hyperlink/test/__pycache__/test_url.cpython-39.pyc new file mode 100644 index 0000000..1453efd Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/hyperlink/test/__pycache__/test_url.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/hyperlink/test/common.py b/WebCrawler/venv/Lib/site-packages/hyperlink/test/common.py new file mode 100644 index 0000000..1eec0db --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/hyperlink/test/common.py @@ -0,0 +1,68 @@ +from typing import Any, Callable, Optional, Type +from unittest import TestCase + + +class HyperlinkTestCase(TestCase): + """This type mostly exists to provide a backwards-compatible + assertRaises method for Python 2.6 testing. + """ + + def assertRaises( # type: ignore[override] + self, + expected_exception, # type: Type[BaseException] + callableObj=None, # type: Optional[Callable[..., Any]] + *args, # type: Any + **kwargs # type: Any + ): + # type: (...) -> Any + """Fail unless an exception of class expected_exception is raised + by callableObj when invoked with arguments args and keyword + arguments kwargs. If a different type of exception is + raised, it will not be caught, and the test case will be + deemed to have suffered an error, exactly as for an + unexpected exception. + + If called with callableObj omitted or None, will return a + context object used like this:: + + with self.assertRaises(SomeException): + do_something() + + The context manager keeps a reference to the exception as + the 'exception' attribute. This allows you to inspect the + exception after the assertion:: + + with self.assertRaises(SomeException) as cm: + do_something() + the_exception = cm.exception + self.assertEqual(the_exception.error_code, 3) + """ + context = _AssertRaisesContext(expected_exception, self) + if callableObj is None: + return context + with context: + callableObj(*args, **kwargs) + + +class _AssertRaisesContext(object): + "A context manager used to implement HyperlinkTestCase.assertRaises." + + def __init__(self, expected, test_case): + # type: (Type[BaseException], TestCase) -> None + self.expected = expected + self.failureException = test_case.failureException + + def __enter__(self): + # type: () -> "_AssertRaisesContext" + return self + + def __exit__(self, exc_type, exc_value, tb): + # type: (Optional[Type[BaseException]], Any, Any) -> bool + if exc_type is None: + exc_name = self.expected.__name__ + raise self.failureException("%s not raised" % (exc_name,)) + if not issubclass(exc_type, self.expected): + # let unexpected exceptions pass through + return False + self.exception = exc_value # store for later retrieval + return True diff --git a/WebCrawler/venv/Lib/site-packages/hyperlink/test/test_common.py b/WebCrawler/venv/Lib/site-packages/hyperlink/test/test_common.py new file mode 100644 index 0000000..6827d0b --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/hyperlink/test/test_common.py @@ -0,0 +1,120 @@ +""" +Tests for hyperlink.test.common +""" +from typing import Any +from unittest import TestCase +from .common import HyperlinkTestCase + + +class _ExpectedException(Exception): + """An exception used to test HyperlinkTestCase.assertRaises. + + """ + + +class _UnexpectedException(Exception): + """An exception used to test HyperlinkTestCase.assertRaises. + + """ + + +class TestHyperlink(TestCase): + """Tests for HyperlinkTestCase""" + + def setUp(self): + # type: () -> None + self.hyperlink_test = HyperlinkTestCase("run") + + def test_assertRaisesWithCallable(self): + # type: () -> None + """HyperlinkTestCase.assertRaises does not raise an AssertionError + when given a callable that, when called with the provided + arguments, raises the expected exception. + + """ + called_with = [] + + def raisesExpected(*args, **kwargs): + # type: (Any, Any) -> None + called_with.append((args, kwargs)) + raise _ExpectedException + + self.hyperlink_test.assertRaises( + _ExpectedException, raisesExpected, 1, keyword=True + ) + self.assertEqual(called_with, [((1,), {"keyword": True})]) + + def test_assertRaisesWithCallableUnexpectedException(self): + # type: () -> None + """When given a callable that raises an unexpected exception, + HyperlinkTestCase.assertRaises raises that exception. + + """ + + def doesNotRaiseExpected(*args, **kwargs): + # type: (Any, Any) -> None + raise _UnexpectedException + + try: + self.hyperlink_test.assertRaises( + _ExpectedException, doesNotRaiseExpected + ) + except _UnexpectedException: + pass + + def test_assertRaisesWithCallableDoesNotRaise(self): + # type: () -> None + """HyperlinkTestCase.assertRaises raises an AssertionError when given + a callable that, when called, does not raise any exception. + + """ + + def doesNotRaise(*args, **kwargs): + # type: (Any, Any) -> None + pass + + try: + self.hyperlink_test.assertRaises(_ExpectedException, doesNotRaise) + except AssertionError: + pass + + def test_assertRaisesContextManager(self): + # type: () -> None + """HyperlinkTestCase.assertRaises does not raise an AssertionError + when used as a context manager with a suite that raises the + expected exception. The context manager stores the exception + instance under its `exception` instance variable. + + """ + with self.hyperlink_test.assertRaises(_ExpectedException) as cm: + raise _ExpectedException + + self.assertTrue( # type: ignore[unreachable] + isinstance(cm.exception, _ExpectedException) + ) + + def test_assertRaisesContextManagerUnexpectedException(self): + # type: () -> None + """When used as a context manager with a block that raises an + unexpected exception, HyperlinkTestCase.assertRaises raises + that unexpected exception. + + """ + try: + with self.hyperlink_test.assertRaises(_ExpectedException): + raise _UnexpectedException + except _UnexpectedException: + pass + + def test_assertRaisesContextManagerDoesNotRaise(self): + # type: () -> None + """HyperlinkTestcase.assertRaises raises an AssertionError when used + as a context manager with a block that does not raise any + exception. + + """ + try: + with self.hyperlink_test.assertRaises(_ExpectedException): + pass + except AssertionError: + pass diff --git a/WebCrawler/venv/Lib/site-packages/hyperlink/test/test_decoded_url.py b/WebCrawler/venv/Lib/site-packages/hyperlink/test/test_decoded_url.py new file mode 100644 index 0000000..7104bea --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/hyperlink/test/test_decoded_url.py @@ -0,0 +1,212 @@ +# -*- coding: utf-8 -*- + +from __future__ import unicode_literals + +from typing import Dict, Union +from .. import DecodedURL, URL +from .._url import _percent_decode +from .common import HyperlinkTestCase + +BASIC_URL = "http://example.com/#" +TOTAL_URL = ( + "https://%75%73%65%72:%00%00%00%00@xn--bcher-kva.ch:8080/" + "a/nice%20nice/./path/?zot=23%25&zut#frég" +) + + +class TestURL(HyperlinkTestCase): + def test_durl_basic(self): + # type: () -> None + bdurl = DecodedURL.from_text(BASIC_URL) + assert bdurl.scheme == "http" + assert bdurl.host == "example.com" + assert bdurl.port == 80 + assert bdurl.path == ("",) + assert bdurl.fragment == "" + + durl = DecodedURL.from_text(TOTAL_URL) + + assert durl.scheme == "https" + assert durl.host == "bücher.ch" + assert durl.port == 8080 + assert durl.path == ("a", "nice nice", ".", "path", "") + assert durl.fragment == "frég" + assert durl.get("zot") == ["23%"] + + assert durl.user == "user" + assert durl.userinfo == ("user", "\0\0\0\0") + + def test_passthroughs(self): + # type: () -> None + + # just basic tests for the methods that more or less pass straight + # through to the underlying URL + + durl = DecodedURL.from_text(TOTAL_URL) + assert durl.sibling("te%t").path[-1] == "te%t" + assert durl.child("../test2%").path[-1] == "../test2%" + assert durl.child() == durl + assert durl.child() is durl + assert durl.click("/").path[-1] == "" + assert durl.user == "user" + + assert "." in durl.path + assert "." not in durl.normalize().path + + assert durl.to_uri().fragment == "fr%C3%A9g" + assert " " in durl.to_iri().path[1] + + assert durl.to_text(with_password=True) == TOTAL_URL + + assert durl.absolute + assert durl.rooted + + assert durl == durl.encoded_url.get_decoded_url() + + durl2 = DecodedURL.from_text(TOTAL_URL, lazy=True) + assert durl2 == durl2.encoded_url.get_decoded_url(lazy=True) + + assert ( + str(DecodedURL.from_text(BASIC_URL).child(" ")) + == "http://example.com/%20" + ) + + assert not (durl == 1) + assert durl != 1 + + def test_repr(self): + # type: () -> None + durl = DecodedURL.from_text(TOTAL_URL) + assert repr(durl) == "DecodedURL(url=" + repr(durl._url) + ")" + + def test_query_manipulation(self): + # type: () -> None + durl = DecodedURL.from_text(TOTAL_URL) + + assert durl.get("zot") == ["23%"] + durl = durl.add(" ", "space") + assert durl.get(" ") == ["space"] + durl = durl.set(" ", "spa%ed") + assert durl.get(" ") == ["spa%ed"] + + durl = DecodedURL(url=durl.to_uri()) + assert durl.get(" ") == ["spa%ed"] + durl = durl.remove(" ") + assert durl.get(" ") == [] + + durl = DecodedURL.from_text("/?%61rg=b&arg=c") + assert durl.get("arg") == ["b", "c"] + + assert durl.set("arg", "d").get("arg") == ["d"] + + durl = DecodedURL.from_text( + "https://example.com/a/b/?fóó=1&bar=2&fóó=3" + ) + assert durl.remove("fóó") == DecodedURL.from_text( + "https://example.com/a/b/?bar=2" + ) + assert durl.remove("fóó", value="1") == DecodedURL.from_text( + "https://example.com/a/b/?bar=2&fóó=3" + ) + assert durl.remove("fóó", limit=1) == DecodedURL.from_text( + "https://example.com/a/b/?bar=2&fóó=3" + ) + assert durl.remove("fóó", value="1", limit=0) == DecodedURL.from_text( + "https://example.com/a/b/?fóó=1&bar=2&fóó=3" + ) + + def test_equality_and_hashability(self): + # type: () -> None + durl = DecodedURL.from_text(TOTAL_URL) + durl2 = DecodedURL.from_text(TOTAL_URL) + burl = DecodedURL.from_text(BASIC_URL) + durl_uri = durl.to_uri() + + assert durl == durl + assert durl == durl2 + assert durl != burl + assert durl is not None + assert durl != durl._url + + AnyURL = Union[URL, DecodedURL] + + durl_map = {} # type: Dict[AnyURL, AnyURL] + durl_map[durl] = durl + durl_map[durl2] = durl2 + + assert len(durl_map) == 1 + + durl_map[burl] = burl + + assert len(durl_map) == 2 + + durl_map[durl_uri] = durl_uri + + assert len(durl_map) == 3 + + def test_replace_roundtrip(self): + # type: () -> None + durl = DecodedURL.from_text(TOTAL_URL) + + durl2 = durl.replace( + scheme=durl.scheme, + host=durl.host, + path=durl.path, + query=durl.query, + fragment=durl.fragment, + port=durl.port, + rooted=durl.rooted, + userinfo=durl.userinfo, + uses_netloc=durl.uses_netloc, + ) + + assert durl == durl2 + + def test_replace_userinfo(self): + # type: () -> None + durl = DecodedURL.from_text(TOTAL_URL) + with self.assertRaises(ValueError): + durl.replace( + userinfo=( # type: ignore[arg-type] + "user", + "pw", + "thiswillcauseafailure", + ) + ) + return + + def test_twisted_compat(self): + # type: () -> None + durl = DecodedURL.from_text(TOTAL_URL) + + assert durl == DecodedURL.fromText(TOTAL_URL) + assert "to_text" in dir(durl) + assert "asText" not in dir(durl) + assert durl.to_text() == durl.asText() + + def test_percent_decode_mixed(self): + # type: () -> None + + # See https://github.com/python-hyper/hyperlink/pull/59 for a + # nice discussion of the possibilities + assert _percent_decode("abcdé%C3%A9éfg") == "abcdéééfg" + + # still allow percent encoding in the case of an error + assert _percent_decode("abcdé%C3éfg") == "abcdé%C3éfg" + + # ...unless explicitly told otherwise + with self.assertRaises(UnicodeDecodeError): + _percent_decode("abcdé%C3éfg", raise_subencoding_exc=True) + + # when not encodable as subencoding + assert _percent_decode("é%25é", subencoding="ascii") == "é%25é" + + def test_click_decoded_url(self): + # type: () -> None + durl = DecodedURL.from_text(TOTAL_URL) + durl_dest = DecodedURL.from_text("/tëst") + + clicked = durl.click(durl_dest) + assert clicked.host == durl.host + assert clicked.path == durl_dest.path + assert clicked.path == ("tëst",) diff --git a/WebCrawler/venv/Lib/site-packages/hyperlink/test/test_hypothesis.py b/WebCrawler/venv/Lib/site-packages/hyperlink/test/test_hypothesis.py new file mode 100644 index 0000000..776ed7b --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/hyperlink/test/test_hypothesis.py @@ -0,0 +1,214 @@ +# -*- coding: utf-8 -*- +""" +Tests for hyperlink.hypothesis. +""" + +try: + import hypothesis + + del hypothesis +except ImportError: + pass +else: + from string import digits + from typing import Sequence, Text + + try: + from unittest.mock import patch + except ImportError: + from mock import patch # type: ignore[misc] + + from hypothesis import given, settings + from hypothesis.strategies import SearchStrategy, data + + from idna import IDNAError, check_label, encode as idna_encode + + from .common import HyperlinkTestCase + from .. import DecodedURL, EncodedURL + from ..hypothesis import ( + DrawCallable, + composite, + decoded_urls, + encoded_urls, + hostname_labels, + hostnames, + idna_text, + paths, + port_numbers, + ) + + class TestHypothesisStrategies(HyperlinkTestCase): + """ + Tests for hyperlink.hypothesis. + """ + + @given(idna_text()) + def test_idna_text_valid(self, text): + # type: (Text) -> None + """ + idna_text() generates IDNA-encodable text. + """ + try: + idna_encode(text) + except IDNAError: # pragma: no cover + raise AssertionError("Invalid IDNA text: {!r}".format(text)) + + @given(data()) + def test_idna_text_min_max(self, data): + # type: (SearchStrategy) -> None + """ + idna_text() raises AssertionError if min_size is < 1. + """ + self.assertRaises(AssertionError, data.draw, idna_text(min_size=0)) + self.assertRaises(AssertionError, data.draw, idna_text(max_size=0)) + + @given(port_numbers()) + def test_port_numbers_bounds(self, port): + # type: (int) -> None + """ + port_numbers() generates integers between 1 and 65535, inclusive. + """ + self.assertGreaterEqual(port, 1) + self.assertLessEqual(port, 65535) + + @given(port_numbers(allow_zero=True)) + def test_port_numbers_bounds_allow_zero(self, port): + # type: (int) -> None + """ + port_numbers(allow_zero=True) generates integers between 0 and + 65535, inclusive. + """ + self.assertGreaterEqual(port, 0) + self.assertLessEqual(port, 65535) + + @given(hostname_labels()) + def test_hostname_labels_valid_idn(self, label): + # type: (Text) -> None + """ + hostname_labels() generates IDN host name labels. + """ + try: + check_label(label) + idna_encode(label) + except UnicodeError: # pragma: no cover + raise AssertionError("Invalid IDN label: {!r}".format(label)) + + @given(data()) + @settings(max_examples=10) + def test_hostname_labels_long_idn_punycode(self, data): + # type: (SearchStrategy) -> None + """ + hostname_labels() handles case where idna_text() generates text + that encoded to punycode ends up as longer than allowed. + """ + + @composite + def mock_idna_text(draw, min_size, max_size): + # type: (DrawCallable, int, int) -> Text + # We want a string that does not exceed max_size, but when + # encoded to punycode, does exceed max_size. + # So use a unicode character that is larger when encoded, + # "á" being a great example, and use it max_size times, which + # will be max_size * 3 in size when encoded. + return u"\N{LATIN SMALL LETTER A WITH ACUTE}" * max_size + + with patch("hyperlink.hypothesis.idna_text", mock_idna_text): + label = data.draw(hostname_labels()) + try: + check_label(label) + idna_encode(label) + except UnicodeError: # pragma: no cover + raise AssertionError( + "Invalid IDN label: {!r}".format(label) + ) + + @given(hostname_labels(allow_idn=False)) + def test_hostname_labels_valid_ascii(self, label): + # type: (Text) -> None + """ + hostname_labels() generates a ASCII host name labels. + """ + try: + check_label(label) + label.encode("ascii") + except UnicodeError: # pragma: no cover + raise AssertionError("Invalid ASCII label: {!r}".format(label)) + + @given(hostnames()) + def test_hostnames_idn(self, hostname): + # type: (Text) -> None + """ + hostnames() generates a IDN host names. + """ + try: + for label in hostname.split(u"."): + check_label(label) + idna_encode(hostname) + except UnicodeError: # pragma: no cover + raise AssertionError( + "Invalid IDN host name: {!r}".format(hostname) + ) + + @given(hostnames(allow_leading_digit=False)) + def test_hostnames_idn_nolead(self, hostname): + # type: (Text) -> None + """ + hostnames(allow_leading_digit=False) generates a IDN host names + without leading digits. + """ + self.assertTrue(hostname == hostname.lstrip(digits)) + + @given(hostnames(allow_idn=False)) + def test_hostnames_ascii(self, hostname): + # type: (Text) -> None + """ + hostnames() generates a ASCII host names. + """ + try: + for label in hostname.split(u"."): + check_label(label) + hostname.encode("ascii") + except UnicodeError: # pragma: no cover + raise AssertionError( + "Invalid ASCII host name: {!r}".format(hostname) + ) + + @given(hostnames(allow_leading_digit=False, allow_idn=False)) + def test_hostnames_ascii_nolead(self, hostname): + # type: (Text) -> None + """ + hostnames(allow_leading_digit=False, allow_idn=False) generates + ASCII host names without leading digits. + """ + self.assertTrue(hostname == hostname.lstrip(digits)) + + @given(paths()) + def test_paths(self, path): + # type: (Sequence[Text]) -> None + """ + paths() generates sequences of URL path components. + """ + text = u"/".join(path) + try: + text.encode("utf-8") + except UnicodeError: # pragma: no cover + raise AssertionError("Invalid URL path: {!r}".format(path)) + + for segment in path: + self.assertNotIn("#/?", segment) + + @given(encoded_urls()) + def test_encoded_urls(self, url): + # type: (EncodedURL) -> None + """ + encoded_urls() generates EncodedURLs. + """ + self.assertIsInstance(url, EncodedURL) + + @given(decoded_urls()) + def test_decoded_urls(self, url): + # type: (DecodedURL) -> None + """ + decoded_urls() generates DecodedURLs. + """ + self.assertIsInstance(url, DecodedURL) diff --git a/WebCrawler/venv/Lib/site-packages/hyperlink/test/test_parse.py b/WebCrawler/venv/Lib/site-packages/hyperlink/test/test_parse.py new file mode 100644 index 0000000..66b0270 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/hyperlink/test/test_parse.py @@ -0,0 +1,37 @@ +# -*- coding: utf-8 -*- + +from __future__ import unicode_literals + +from .common import HyperlinkTestCase +from hyperlink import parse, EncodedURL, DecodedURL + +BASIC_URL = "http://example.com/#" +TOTAL_URL = ( + "https://%75%73%65%72:%00%00%00%00@xn--bcher-kva.ch:8080" + "/a/nice%20nice/./path/?zot=23%25&zut#frég" +) +UNDECODABLE_FRAG_URL = TOTAL_URL + "%C3" +# the %C3 above percent-decodes to an unpaired \xc3 byte which makes this +# invalid utf8 + + +class TestURL(HyperlinkTestCase): + def test_parse(self): + # type: () -> None + purl = parse(TOTAL_URL) + assert isinstance(purl, DecodedURL) + assert purl.user == "user" + assert purl.get("zot") == ["23%"] + assert purl.fragment == "frég" + + purl2 = parse(TOTAL_URL, decoded=False) + assert isinstance(purl2, EncodedURL) + assert purl2.get("zot") == ["23%25"] + + with self.assertRaises(UnicodeDecodeError): + purl3 = parse(UNDECODABLE_FRAG_URL) + + purl3 = parse(UNDECODABLE_FRAG_URL, lazy=True) + + with self.assertRaises(UnicodeDecodeError): + purl3.fragment diff --git a/WebCrawler/venv/Lib/site-packages/hyperlink/test/test_scheme_registration.py b/WebCrawler/venv/Lib/site-packages/hyperlink/test/test_scheme_registration.py new file mode 100644 index 0000000..f98109a --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/hyperlink/test/test_scheme_registration.py @@ -0,0 +1,72 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals +from typing import cast + + +from .. import _url +from .common import HyperlinkTestCase +from .._url import register_scheme, URL + + +class TestSchemeRegistration(HyperlinkTestCase): + def setUp(self): + # type: () -> None + self._orig_scheme_port_map = dict(_url.SCHEME_PORT_MAP) + self._orig_no_netloc_schemes = set(_url.NO_NETLOC_SCHEMES) + + def tearDown(self): + # type: () -> None + _url.SCHEME_PORT_MAP = self._orig_scheme_port_map + _url.NO_NETLOC_SCHEMES = self._orig_no_netloc_schemes + + def test_register_scheme_basic(self): + # type: () -> None + register_scheme("deltron", uses_netloc=True, default_port=3030) + + u1 = URL.from_text("deltron://example.com") + assert u1.scheme == "deltron" + assert u1.port == 3030 + assert u1.uses_netloc is True + + # test netloc works even when the original gives no indication + u2 = URL.from_text("deltron:") + u2 = u2.replace(host="example.com") + assert u2.to_text() == "deltron://example.com" + + # test default port means no emission + u3 = URL.from_text("deltron://example.com:3030") + assert u3.to_text() == "deltron://example.com" + + register_scheme("nonetron", default_port=3031) + u4 = URL(scheme="nonetron") + u4 = u4.replace(host="example.com") + assert u4.to_text() == "nonetron://example.com" + + def test_register_no_netloc_scheme(self): + # type: () -> None + register_scheme("noloctron", uses_netloc=False) + u4 = URL(scheme="noloctron") + u4 = u4.replace(path=("example", "path")) + assert u4.to_text() == "noloctron:example/path" + + def test_register_no_netloc_with_port(self): + # type: () -> None + with self.assertRaises(ValueError): + register_scheme("badnetlocless", uses_netloc=False, default_port=7) + + def test_invalid_uses_netloc(self): + # type: () -> None + with self.assertRaises(ValueError): + register_scheme("badnetloc", uses_netloc=cast(bool, None)) + with self.assertRaises(ValueError): + register_scheme("badnetloc", uses_netloc=cast(bool, object())) + + def test_register_invalid_uses_netloc(self): + # type: () -> None + with self.assertRaises(ValueError): + register_scheme("lol", uses_netloc=cast(bool, object())) + + def test_register_invalid_port(self): + # type: () -> None + with self.assertRaises(ValueError): + register_scheme("nope", default_port=cast(bool, object())) diff --git a/WebCrawler/venv/Lib/site-packages/hyperlink/test/test_socket.py b/WebCrawler/venv/Lib/site-packages/hyperlink/test/test_socket.py new file mode 100644 index 0000000..5f83d45 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/hyperlink/test/test_socket.py @@ -0,0 +1,45 @@ +# mypy: always-true=inet_pton + +try: + from socket import inet_pton +except ImportError: + inet_pton = None # type: ignore[assignment] + +if not inet_pton: + import socket + + from .common import HyperlinkTestCase + from .._socket import inet_pton + + class TestSocket(HyperlinkTestCase): + def test_inet_pton_ipv4_valid(self): + # type: () -> None + data = inet_pton(socket.AF_INET, "127.0.0.1") + assert isinstance(data, bytes) + + def test_inet_pton_ipv4_bogus(self): + # type: () -> None + with self.assertRaises(socket.error): + inet_pton(socket.AF_INET, "blah") + + def test_inet_pton_ipv6_valid(self): + # type: () -> None + data = inet_pton(socket.AF_INET6, "::1") + assert isinstance(data, bytes) + + def test_inet_pton_ipv6_bogus(self): + # type: () -> None + with self.assertRaises(socket.error): + inet_pton(socket.AF_INET6, "blah") + + def test_inet_pton_bogus_family(self): + # type: () -> None + # Find an integer not associated with a known address family + i = int(socket.AF_INET6) + while True: + if i != socket.AF_INET and i != socket.AF_INET6: + break + i += 100 + + with self.assertRaises(socket.error): + inet_pton(i, "127.0.0.1") diff --git a/WebCrawler/venv/Lib/site-packages/hyperlink/test/test_url.py b/WebCrawler/venv/Lib/site-packages/hyperlink/test/test_url.py new file mode 100644 index 0000000..159d6a5 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/hyperlink/test/test_url.py @@ -0,0 +1,1493 @@ +# -*- coding: utf-8 -*- + +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +from __future__ import unicode_literals + +import sys +import socket +from typing import Any, Iterable, Optional, Text, Tuple, cast + +from .common import HyperlinkTestCase +from .. import URL, URLParseError +from .._url import inet_pton, SCHEME_PORT_MAP + + +PY2 = sys.version_info[0] == 2 +unicode = type("") + + +BASIC_URL = "http://www.foo.com/a/nice/path/?zot=23&zut" + +# Examples from RFC 3986 section 5.4, Reference Resolution Examples +relativeLinkBaseForRFC3986 = "http://a/b/c/d;p?q" +relativeLinkTestsForRFC3986 = [ + # "Normal" + # ('g:h', 'g:h'), # can't click on a scheme-having url without an abs path + ("g", "http://a/b/c/g"), + ("./g", "http://a/b/c/g"), + ("g/", "http://a/b/c/g/"), + ("/g", "http://a/g"), + ("//g", "http://g"), + ("?y", "http://a/b/c/d;p?y"), + ("g?y", "http://a/b/c/g?y"), + ("#s", "http://a/b/c/d;p?q#s"), + ("g#s", "http://a/b/c/g#s"), + ("g?y#s", "http://a/b/c/g?y#s"), + (";x", "http://a/b/c/;x"), + ("g;x", "http://a/b/c/g;x"), + ("g;x?y#s", "http://a/b/c/g;x?y#s"), + ("", "http://a/b/c/d;p?q"), + (".", "http://a/b/c/"), + ("./", "http://a/b/c/"), + ("..", "http://a/b/"), + ("../", "http://a/b/"), + ("../g", "http://a/b/g"), + ("../..", "http://a/"), + ("../../", "http://a/"), + ("../../g", "http://a/g"), + # Abnormal examples + # ".." cannot be used to change the authority component of a URI. + ("../../../g", "http://a/g"), + ("../../../../g", "http://a/g"), + # Only include "." and ".." when they are only part of a larger segment, + # not by themselves. + ("/./g", "http://a/g"), + ("/../g", "http://a/g"), + ("g.", "http://a/b/c/g."), + (".g", "http://a/b/c/.g"), + ("g..", "http://a/b/c/g.."), + ("..g", "http://a/b/c/..g"), + # Unnecessary or nonsensical forms of "." and "..". + ("./../g", "http://a/b/g"), + ("./g/.", "http://a/b/c/g/"), + ("g/./h", "http://a/b/c/g/h"), + ("g/../h", "http://a/b/c/h"), + ("g;x=1/./y", "http://a/b/c/g;x=1/y"), + ("g;x=1/../y", "http://a/b/c/y"), + # Separating the reference's query and fragment components from the path. + ("g?y/./x", "http://a/b/c/g?y/./x"), + ("g?y/../x", "http://a/b/c/g?y/../x"), + ("g#s/./x", "http://a/b/c/g#s/./x"), + ("g#s/../x", "http://a/b/c/g#s/../x"), +] + + +ROUNDTRIP_TESTS = ( + "http://localhost", + "http://localhost/", + "http://127.0.0.1/", + "http://[::127.0.0.1]/", + "http://[::1]/", + "http://localhost/foo", + "http://localhost/foo/", + "http://localhost/foo!!bar/", + "http://localhost/foo%20bar/", + "http://localhost/foo%2Fbar/", + "http://localhost/foo?n", + "http://localhost/foo?n=v", + "http://localhost/foo?n=/a/b", + "http://example.com/foo!@$bar?b!@z=123", + "http://localhost/asd?a=asd%20sdf/345", + "http://(%2525)/(%2525)?(%2525)&(%2525)=(%2525)#(%2525)", + "http://(%C3%A9)/(%C3%A9)?(%C3%A9)&(%C3%A9)=(%C3%A9)#(%C3%A9)", + "?sslrootcert=/Users/glyph/Downloads/rds-ca-2015-root.pem&sslmode=verify", + # from boltons.urlutils' tests + "http://googlewebsite.com/e-shops.aspx", + "http://example.com:8080/search?q=123&business=Nothing%20Special", + "http://hatnote.com:9000/?arg=1&arg=2&arg=3", + "https://xn--bcher-kva.ch", + "http://xn--ggbla1c4e.xn--ngbc5azd/", + "http://tools.ietf.org/html/rfc3986#section-3.4", + # 'http://wiki:pedia@hatnote.com', + "ftp://ftp.rfc-editor.org/in-notes/tar/RFCs0001-0500.tar.gz", + "http://[1080:0:0:0:8:800:200C:417A]/index.html", + "ssh://192.0.2.16:2222/", + "https://[::101.45.75.219]:80/?hi=bye", + "ldap://[::192.9.5.5]/dc=example,dc=com??sub?(sn=Jensen)", + "mailto:me@example.com?to=me@example.com&body=hi%20http://wikipedia.org", + "news:alt.rec.motorcycle", + "tel:+1-800-867-5309", + "urn:oasis:member:A00024:x", + ( + "magnet:?xt=urn:btih:1a42b9e04e122b97a5254e3df77ab3c4b7da725f&dn=Puppy%" + "20Linux%20precise-5.7.1.iso&tr=udp://tracker.openbittorrent.com:80&" + "tr=udp://tracker.publicbt.com:80&tr=udp://tracker.istole.it:6969&" + "tr=udp://tracker.ccc.de:80&tr=udp://open.demonii.com:1337" + ), + # percent-encoded delimiters in percent-encodable fields + "https://%3A@example.com/", # colon in username + "https://%40@example.com/", # at sign in username + "https://%2f@example.com/", # slash in username + "https://a:%3a@example.com/", # colon in password + "https://a:%40@example.com/", # at sign in password + "https://a:%2f@example.com/", # slash in password + "https://a:%3f@example.com/", # question mark in password + "https://example.com/%2F/", # slash in path + "https://example.com/%3F/", # question mark in path + "https://example.com/%23/", # hash in path + "https://example.com/?%23=b", # hash in query param name + "https://example.com/?%3D=b", # equals in query param name + "https://example.com/?%26=b", # ampersand in query param name + "https://example.com/?a=%23", # hash in query param value + "https://example.com/?a=%26", # ampersand in query param value + "https://example.com/?a=%3D", # equals in query param value + # double-encoded percent sign in all percent-encodable positions: + "http://(%2525):(%2525)@example.com/(%2525)/?(%2525)=(%2525)#(%2525)", + # colon in first part of schemeless relative url + "first_seg_rel_path__colon%3Anotok/second_seg__colon%3Aok", +) + + +class TestURL(HyperlinkTestCase): + """ + Tests for L{URL}. + """ + + def assertUnicoded(self, u): + # type: (URL) -> None + """ + The given L{URL}'s components should be L{unicode}. + + @param u: The L{URL} to test. + """ + self.assertTrue( + isinstance(u.scheme, unicode) or u.scheme is None, repr(u) + ) + self.assertTrue(isinstance(u.host, unicode) or u.host is None, repr(u)) + for seg in u.path: + self.assertEqual(type(seg), unicode, repr(u)) + for (_k, v) in u.query: + self.assertEqual(type(seg), unicode, repr(u)) + self.assertTrue(v is None or isinstance(v, unicode), repr(u)) + self.assertEqual(type(u.fragment), unicode, repr(u)) + + def assertURL( + self, + u, # type: URL + scheme, # type: Text + host, # type: Text + path, # type: Iterable[Text] + query, # type: Iterable[Tuple[Text, Optional[Text]]] + fragment, # type: Text + port, # type: Optional[int] + userinfo="", # type: Text + ): + # type: (...) -> None + """ + The given L{URL} should have the given components. + + @param u: The actual L{URL} to examine. + + @param scheme: The expected scheme. + + @param host: The expected host. + + @param path: The expected path. + + @param query: The expected query. + + @param fragment: The expected fragment. + + @param port: The expected port. + + @param userinfo: The expected userinfo. + """ + actual = ( + u.scheme, + u.host, + u.path, + u.query, + u.fragment, + u.port, + u.userinfo, + ) + expected = ( + scheme, + host, + tuple(path), + tuple(query), + fragment, + port, + u.userinfo, + ) + self.assertEqual(actual, expected) + + def test_initDefaults(self): + # type: () -> None + """ + L{URL} should have appropriate default values. + """ + + def check(u): + # type: (URL) -> None + self.assertUnicoded(u) + self.assertURL(u, "http", "", [], [], "", 80, "") + + check(URL("http", "")) + check(URL("http", "", [], [])) + check(URL("http", "", [], [], "")) + + def test_init(self): + # type: () -> None + """ + L{URL} should accept L{unicode} parameters. + """ + u = URL("s", "h", ["p"], [("k", "v"), ("k", None)], "f") + self.assertUnicoded(u) + self.assertURL(u, "s", "h", ["p"], [("k", "v"), ("k", None)], "f", None) + + self.assertURL( + URL("http", "\xe0", ["\xe9"], [("\u03bb", "\u03c0")], "\u22a5"), + "http", + "\xe0", + ["\xe9"], + [("\u03bb", "\u03c0")], + "\u22a5", + 80, + ) + + def test_initPercent(self): + # type: () -> None + """ + L{URL} should accept (and not interpret) percent characters. + """ + u = URL("s", "%68", ["%70"], [("%6B", "%76"), ("%6B", None)], "%66") + self.assertUnicoded(u) + self.assertURL( + u, "s", "%68", ["%70"], [("%6B", "%76"), ("%6B", None)], "%66", None + ) + + def test_repr(self): + # type: () -> None + """ + L{URL.__repr__} will display the canonical form of the URL, wrapped in + a L{URL.from_text} invocation, so that it is C{eval}-able but still + easy to read. + """ + self.assertEqual( + repr( + URL( + scheme="http", + host="foo", + path=["bar"], + query=[("baz", None), ("k", "v")], + fragment="frob", + ) + ), + "URL.from_text(%s)" % (repr("http://foo/bar?baz&k=v#frob"),), + ) + + def test_from_text(self): + # type: () -> None + """ + Round-tripping L{URL.from_text} with C{str} results in an equivalent + URL. + """ + urlpath = URL.from_text(BASIC_URL) + self.assertEqual(BASIC_URL, urlpath.to_text()) + + def test_roundtrip(self): + # type: () -> None + """ + L{URL.to_text} should invert L{URL.from_text}. + """ + for test in ROUNDTRIP_TESTS: + result = URL.from_text(test).to_text(with_password=True) + self.assertEqual(test, result) + + def test_roundtrip_double_iri(self): + # type: () -> None + for test in ROUNDTRIP_TESTS: + url = URL.from_text(test) + iri = url.to_iri() + double_iri = iri.to_iri() + assert iri == double_iri + + iri_text = iri.to_text(with_password=True) + double_iri_text = double_iri.to_text(with_password=True) + assert iri_text == double_iri_text + return + + def test_equality(self): + # type: () -> None + """ + Two URLs decoded using L{URL.from_text} will be equal (C{==}) if they + decoded same URL string, and unequal (C{!=}) if they decoded different + strings. + """ + urlpath = URL.from_text(BASIC_URL) + self.assertEqual(urlpath, URL.from_text(BASIC_URL)) + self.assertNotEqual( + urlpath, + URL.from_text( + "ftp://www.anotherinvaliddomain.com/" "foo/bar/baz/?zot=21&zut" + ), + ) + + def test_fragmentEquality(self): + # type: () -> None + """ + An URL created with the empty string for a fragment compares equal + to an URL created with an unspecified fragment. + """ + self.assertEqual(URL(fragment=""), URL()) + self.assertEqual( + URL.from_text("http://localhost/#"), + URL.from_text("http://localhost/"), + ) + + def test_child(self): + # type: () -> None + """ + L{URL.child} appends a new path segment, but does not affect the query + or fragment. + """ + urlpath = URL.from_text(BASIC_URL) + self.assertEqual( + "http://www.foo.com/a/nice/path/gong?zot=23&zut", + urlpath.child("gong").to_text(), + ) + self.assertEqual( + "http://www.foo.com/a/nice/path/gong%2F?zot=23&zut", + urlpath.child("gong/").to_text(), + ) + self.assertEqual( + "http://www.foo.com/a/nice/path/gong%2Fdouble?zot=23&zut", + urlpath.child("gong/double").to_text(), + ) + self.assertEqual( + "http://www.foo.com/a/nice/path/gong%2Fdouble%2F?zot=23&zut", + urlpath.child("gong/double/").to_text(), + ) + + def test_multiChild(self): + # type: () -> None + """ + L{URL.child} receives multiple segments as C{*args} and appends each in + turn. + """ + url = URL.from_text("http://example.com/a/b") + self.assertEqual( + url.child("c", "d", "e").to_text(), "http://example.com/a/b/c/d/e" + ) + + def test_childInitRoot(self): + # type: () -> None + """ + L{URL.child} of a L{URL} without a path produces a L{URL} with a single + path segment. + """ + childURL = URL(host="www.foo.com").child("c") + self.assertTrue(childURL.rooted) + self.assertEqual("http://www.foo.com/c", childURL.to_text()) + + def test_emptyChild(self): + # type: () -> None + """ + L{URL.child} without any new segments returns the original L{URL}. + """ + url = URL(host="www.foo.com") + self.assertEqual(url.child(), url) + + def test_sibling(self): + # type: () -> None + """ + L{URL.sibling} of a L{URL} replaces the last path segment, but does not + affect the query or fragment. + """ + urlpath = URL.from_text(BASIC_URL) + self.assertEqual( + "http://www.foo.com/a/nice/path/sister?zot=23&zut", + urlpath.sibling("sister").to_text(), + ) + # Use an url without trailing '/' to check child removal. + url_text = "http://www.foo.com/a/nice/path?zot=23&zut" + urlpath = URL.from_text(url_text) + self.assertEqual( + "http://www.foo.com/a/nice/sister?zot=23&zut", + urlpath.sibling("sister").to_text(), + ) + + def test_click(self): + # type: () -> None + """ + L{URL.click} interprets the given string as a relative URI-reference + and returns a new L{URL} interpreting C{self} as the base absolute URI. + """ + urlpath = URL.from_text(BASIC_URL) + # A null uri should be valid (return here). + self.assertEqual( + "http://www.foo.com/a/nice/path/?zot=23&zut", + urlpath.click("").to_text(), + ) + # A simple relative path remove the query. + self.assertEqual( + "http://www.foo.com/a/nice/path/click", + urlpath.click("click").to_text(), + ) + # An absolute path replace path and query. + self.assertEqual( + "http://www.foo.com/click", urlpath.click("/click").to_text() + ) + # Replace just the query. + self.assertEqual( + "http://www.foo.com/a/nice/path/?burp", + urlpath.click("?burp").to_text(), + ) + # One full url to another should not generate '//' between authority. + # and path + self.assertTrue( + "//foobar" + not in urlpath.click("http://www.foo.com/foobar").to_text() + ) + + # From a url with no query clicking a url with a query, the query + # should be handled properly. + u = URL.from_text("http://www.foo.com/me/noquery") + self.assertEqual( + "http://www.foo.com/me/17?spam=158", + u.click("/me/17?spam=158").to_text(), + ) + + # Check that everything from the path onward is removed when the click + # link has no path. + u = URL.from_text("http://localhost/foo?abc=def") + self.assertEqual( + u.click("http://www.python.org").to_text(), "http://www.python.org" + ) + + # https://twistedmatrix.com/trac/ticket/8184 + u = URL.from_text("http://hatnote.com/a/b/../c/./d/e/..") + res = "http://hatnote.com/a/c/d/" + self.assertEqual(u.click("").to_text(), res) + + # test click default arg is same as empty string above + self.assertEqual(u.click().to_text(), res) + + # test click on a URL instance + u = URL.fromText("http://localhost/foo/?abc=def") + u2 = URL.from_text("bar") + u3 = u.click(u2) + self.assertEqual(u3.to_text(), "http://localhost/foo/bar") + + def test_clickRFC3986(self): + # type: () -> None + """ + L{URL.click} should correctly resolve the examples in RFC 3986. + """ + base = URL.from_text(relativeLinkBaseForRFC3986) + for (ref, expected) in relativeLinkTestsForRFC3986: + self.assertEqual(base.click(ref).to_text(), expected) + + def test_clickSchemeRelPath(self): + # type: () -> None + """ + L{URL.click} should not accept schemes with relative paths. + """ + base = URL.from_text(relativeLinkBaseForRFC3986) + self.assertRaises(NotImplementedError, base.click, "g:h") + self.assertRaises(NotImplementedError, base.click, "http:h") + + def test_cloneUnchanged(self): + # type: () -> None + """ + Verify that L{URL.replace} doesn't change any of the arguments it + is passed. + """ + urlpath = URL.from_text("https://x:1/y?z=1#A") + self.assertEqual( + urlpath.replace( + urlpath.scheme, + urlpath.host, + urlpath.path, + urlpath.query, + urlpath.fragment, + urlpath.port, + ), + urlpath, + ) + self.assertEqual(urlpath.replace(), urlpath) + + def test_clickCollapse(self): + # type: () -> None + """ + L{URL.click} collapses C{.} and C{..} according to RFC 3986 section + 5.2.4. + """ + tests = [ + ["http://localhost/", ".", "http://localhost/"], + ["http://localhost/", "..", "http://localhost/"], + ["http://localhost/a/b/c", ".", "http://localhost/a/b/"], + ["http://localhost/a/b/c", "..", "http://localhost/a/"], + ["http://localhost/a/b/c", "./d/e", "http://localhost/a/b/d/e"], + ["http://localhost/a/b/c", "../d/e", "http://localhost/a/d/e"], + ["http://localhost/a/b/c", "/./d/e", "http://localhost/d/e"], + ["http://localhost/a/b/c", "/../d/e", "http://localhost/d/e"], + [ + "http://localhost/a/b/c/", + "../../d/e/", + "http://localhost/a/d/e/", + ], + ["http://localhost/a/./c", "../d/e", "http://localhost/d/e"], + ["http://localhost/a/./c/", "../d/e", "http://localhost/a/d/e"], + [ + "http://localhost/a/b/c/d", + "./e/../f/../g", + "http://localhost/a/b/c/g", + ], + ["http://localhost/a/b/c", "d//e", "http://localhost/a/b/d//e"], + ] + for start, click, expected in tests: + actual = URL.from_text(start).click(click).to_text() + self.assertEqual( + actual, + expected, + "{start}.click({click}) => {actual} not {expected}".format( + start=start, + click=repr(click), + actual=actual, + expected=expected, + ), + ) + + def test_queryAdd(self): + # type: () -> None + """ + L{URL.add} adds query parameters. + """ + self.assertEqual( + "http://www.foo.com/a/nice/path/?foo=bar", + URL.from_text("http://www.foo.com/a/nice/path/") + .add("foo", "bar") + .to_text(), + ) + self.assertEqual( + "http://www.foo.com/?foo=bar", + URL(host="www.foo.com").add("foo", "bar").to_text(), + ) + urlpath = URL.from_text(BASIC_URL) + self.assertEqual( + "http://www.foo.com/a/nice/path/?zot=23&zut&burp", + urlpath.add("burp").to_text(), + ) + self.assertEqual( + "http://www.foo.com/a/nice/path/?zot=23&zut&burp=xxx", + urlpath.add("burp", "xxx").to_text(), + ) + self.assertEqual( + "http://www.foo.com/a/nice/path/?zot=23&zut&burp=xxx&zing", + urlpath.add("burp", "xxx").add("zing").to_text(), + ) + # Note the inversion! + self.assertEqual( + "http://www.foo.com/a/nice/path/?zot=23&zut&zing&burp=xxx", + urlpath.add("zing").add("burp", "xxx").to_text(), + ) + # Note the two values for the same name. + self.assertEqual( + "http://www.foo.com/a/nice/path/?zot=23&zut&burp=xxx&zot=32", + urlpath.add("burp", "xxx").add("zot", "32").to_text(), + ) + + def test_querySet(self): + # type: () -> None + """ + L{URL.set} replaces query parameters by name. + """ + urlpath = URL.from_text(BASIC_URL) + self.assertEqual( + "http://www.foo.com/a/nice/path/?zot=32&zut", + urlpath.set("zot", "32").to_text(), + ) + # Replace name without value with name/value and vice-versa. + self.assertEqual( + "http://www.foo.com/a/nice/path/?zot&zut=itworked", + urlpath.set("zot").set("zut", "itworked").to_text(), + ) + # Q: what happens when the query has two values and we replace? + # A: we replace both values with a single one + self.assertEqual( + "http://www.foo.com/a/nice/path/?zot=32&zut", + urlpath.add("zot", "xxx").set("zot", "32").to_text(), + ) + + def test_queryRemove(self): + # type: () -> None + """ + L{URL.remove} removes instances of a query parameter. + """ + url = URL.from_text("https://example.com/a/b/?foo=1&bar=2&foo=3") + self.assertEqual( + url.remove("foo"), URL.from_text("https://example.com/a/b/?bar=2") + ) + + self.assertEqual( + url.remove(name="foo", value="1"), + URL.from_text("https://example.com/a/b/?bar=2&foo=3"), + ) + + self.assertEqual( + url.remove(name="foo", limit=1), + URL.from_text("https://example.com/a/b/?bar=2&foo=3"), + ) + + self.assertEqual( + url.remove(name="foo", value="1", limit=0), + URL.from_text("https://example.com/a/b/?foo=1&bar=2&foo=3"), + ) + + def test_parseEqualSignInParamValue(self): + # type: () -> None + """ + Every C{=}-sign after the first in a query parameter is simply included + in the value of the parameter. + """ + u = URL.from_text("http://localhost/?=x=x=x") + self.assertEqual(u.get(""), ["x=x=x"]) + self.assertEqual(u.to_text(), "http://localhost/?=x=x=x") + u = URL.from_text("http://localhost/?foo=x=x=x&bar=y") + self.assertEqual(u.query, (("foo", "x=x=x"), ("bar", "y"))) + self.assertEqual(u.to_text(), "http://localhost/?foo=x=x=x&bar=y") + + u = URL.from_text( + "https://example.com/?argument=3&argument=4&operator=%3D" + ) + iri = u.to_iri() + self.assertEqual(iri.get("operator"), ["="]) + # assert that the equals is not unnecessarily escaped + self.assertEqual(iri.to_uri().get("operator"), ["="]) + + def test_empty(self): + # type: () -> None + """ + An empty L{URL} should serialize as the empty string. + """ + self.assertEqual(URL().to_text(), "") + + def test_justQueryText(self): + # type: () -> None + """ + An L{URL} with query text should serialize as just query text. + """ + u = URL(query=[("hello", "world")]) + self.assertEqual(u.to_text(), "?hello=world") + + def test_identicalEqual(self): + # type: () -> None + """ + L{URL} compares equal to itself. + """ + u = URL.from_text("http://localhost/") + self.assertEqual(u, u) + + def test_similarEqual(self): + # type: () -> None + """ + URLs with equivalent components should compare equal. + """ + u1 = URL.from_text("http://u@localhost:8080/p/a/t/h?q=p#f") + u2 = URL.from_text("http://u@localhost:8080/p/a/t/h?q=p#f") + self.assertEqual(u1, u2) + + def test_differentNotEqual(self): + # type: () -> None + """ + L{URL}s that refer to different resources are both unequal (C{!=}) and + also not equal (not C{==}). + """ + u1 = URL.from_text("http://localhost/a") + u2 = URL.from_text("http://localhost/b") + self.assertFalse(u1 == u2, "%r != %r" % (u1, u2)) + self.assertNotEqual(u1, u2) + + def test_otherTypesNotEqual(self): + # type: () -> None + """ + L{URL} is not equal (C{==}) to other types. + """ + u = URL.from_text("http://localhost/") + self.assertFalse(u == 42, "URL must not equal a number.") + self.assertFalse(u == object(), "URL must not equal an object.") + self.assertNotEqual(u, 42) + self.assertNotEqual(u, object()) + + def test_identicalNotUnequal(self): + # type: () -> None + """ + Identical L{URL}s are not unequal (C{!=}) to each other. + """ + u = URL.from_text("http://u@localhost:8080/p/a/t/h?q=p#f") + self.assertFalse(u != u, "%r == itself" % u) + + def test_similarNotUnequal(self): + # type: () -> None + """ + Structurally similar L{URL}s are not unequal (C{!=}) to each other. + """ + u1 = URL.from_text("http://u@localhost:8080/p/a/t/h?q=p#f") + u2 = URL.from_text("http://u@localhost:8080/p/a/t/h?q=p#f") + self.assertFalse(u1 != u2, "%r == %r" % (u1, u2)) + + def test_differentUnequal(self): + # type: () -> None + """ + Structurally different L{URL}s are unequal (C{!=}) to each other. + """ + u1 = URL.from_text("http://localhost/a") + u2 = URL.from_text("http://localhost/b") + self.assertTrue(u1 != u2, "%r == %r" % (u1, u2)) + + def test_otherTypesUnequal(self): + # type: () -> None + """ + L{URL} is unequal (C{!=}) to other types. + """ + u = URL.from_text("http://localhost/") + self.assertTrue(u != 42, "URL must differ from a number.") + self.assertTrue(u != object(), "URL must be differ from an object.") + + def test_asURI(self): + # type: () -> None + """ + L{URL.asURI} produces an URI which converts any URI unicode encoding + into pure US-ASCII and returns a new L{URL}. + """ + unicodey = ( + "http://\N{LATIN SMALL LETTER E WITH ACUTE}.com/" + "\N{LATIN SMALL LETTER E}\N{COMBINING ACUTE ACCENT}" + "?\N{LATIN SMALL LETTER A}\N{COMBINING ACUTE ACCENT}=" + "\N{LATIN SMALL LETTER I}\N{COMBINING ACUTE ACCENT}" + "#\N{LATIN SMALL LETTER U}\N{COMBINING ACUTE ACCENT}" + ) + iri = URL.from_text(unicodey) + uri = iri.asURI() + self.assertEqual(iri.host, "\N{LATIN SMALL LETTER E WITH ACUTE}.com") + self.assertEqual( + iri.path[0], "\N{LATIN SMALL LETTER E}\N{COMBINING ACUTE ACCENT}" + ) + self.assertEqual(iri.to_text(), unicodey) + expectedURI = "http://xn--9ca.com/%C3%A9?%C3%A1=%C3%AD#%C3%BA" + actualURI = uri.to_text() + self.assertEqual( + actualURI, expectedURI, "%r != %r" % (actualURI, expectedURI) + ) + + def test_asIRI(self): + # type: () -> None + """ + L{URL.asIRI} decodes any percent-encoded text in the URI, making it + more suitable for reading by humans, and returns a new L{URL}. + """ + asciiish = "http://xn--9ca.com/%C3%A9?%C3%A1=%C3%AD#%C3%BA" + uri = URL.from_text(asciiish) + iri = uri.asIRI() + self.assertEqual(uri.host, "xn--9ca.com") + self.assertEqual(uri.path[0], "%C3%A9") + self.assertEqual(uri.to_text(), asciiish) + expectedIRI = ( + "http://\N{LATIN SMALL LETTER E WITH ACUTE}.com/" + "\N{LATIN SMALL LETTER E WITH ACUTE}" + "?\N{LATIN SMALL LETTER A WITH ACUTE}=" + "\N{LATIN SMALL LETTER I WITH ACUTE}" + "#\N{LATIN SMALL LETTER U WITH ACUTE}" + ) + actualIRI = iri.to_text() + self.assertEqual( + actualIRI, expectedIRI, "%r != %r" % (actualIRI, expectedIRI) + ) + + def test_badUTF8AsIRI(self): + # type: () -> None + """ + Bad UTF-8 in a path segment, query parameter, or fragment results in + that portion of the URI remaining percent-encoded in the IRI. + """ + urlWithBinary = "http://xn--9ca.com/%00%FF/%C3%A9" + uri = URL.from_text(urlWithBinary) + iri = uri.asIRI() + expectedIRI = ( + "http://\N{LATIN SMALL LETTER E WITH ACUTE}.com/" + "%00%FF/" + "\N{LATIN SMALL LETTER E WITH ACUTE}" + ) + actualIRI = iri.to_text() + self.assertEqual( + actualIRI, expectedIRI, "%r != %r" % (actualIRI, expectedIRI) + ) + + def test_alreadyIRIAsIRI(self): + # type: () -> None + """ + A L{URL} composed of non-ASCII text will result in non-ASCII text. + """ + unicodey = ( + "http://\N{LATIN SMALL LETTER E WITH ACUTE}.com/" + "\N{LATIN SMALL LETTER E}\N{COMBINING ACUTE ACCENT}" + "?\N{LATIN SMALL LETTER A}\N{COMBINING ACUTE ACCENT}=" + "\N{LATIN SMALL LETTER I}\N{COMBINING ACUTE ACCENT}" + "#\N{LATIN SMALL LETTER U}\N{COMBINING ACUTE ACCENT}" + ) + iri = URL.from_text(unicodey) + alsoIRI = iri.asIRI() + self.assertEqual(alsoIRI.to_text(), unicodey) + + def test_alreadyURIAsURI(self): + # type: () -> None + """ + A L{URL} composed of encoded text will remain encoded. + """ + expectedURI = "http://xn--9ca.com/%C3%A9?%C3%A1=%C3%AD#%C3%BA" + uri = URL.from_text(expectedURI) + actualURI = uri.asURI().to_text() + self.assertEqual(actualURI, expectedURI) + + def test_userinfo(self): + # type: () -> None + """ + L{URL.from_text} will parse the C{userinfo} portion of the URI + separately from the host and port. + """ + url = URL.from_text( + "http://someuser:somepassword@example.com/some-segment@ignore" + ) + self.assertEqual( + url.authority(True), "someuser:somepassword@example.com" + ) + self.assertEqual(url.authority(False), "someuser:@example.com") + self.assertEqual(url.userinfo, "someuser:somepassword") + self.assertEqual(url.user, "someuser") + self.assertEqual( + url.to_text(), "http://someuser:@example.com/some-segment@ignore" + ) + self.assertEqual( + url.replace(userinfo="someuser").to_text(), + "http://someuser@example.com/some-segment@ignore", + ) + + def test_portText(self): + # type: () -> None + """ + L{URL.from_text} parses custom port numbers as integers. + """ + portURL = URL.from_text("http://www.example.com:8080/") + self.assertEqual(portURL.port, 8080) + self.assertEqual(portURL.to_text(), "http://www.example.com:8080/") + + def test_mailto(self): + # type: () -> None + """ + Although L{URL} instances are mainly for dealing with HTTP, other + schemes (such as C{mailto:}) should work as well. For example, + L{URL.from_text}/L{URL.to_text} round-trips cleanly for a C{mailto:} + URL representing an email address. + """ + self.assertEqual( + URL.from_text("mailto:user@example.com").to_text(), + "mailto:user@example.com", + ) + + def test_httpWithoutHost(self): + # type: () -> None + """ + An HTTP URL without a hostname, but with a path, should also round-trip + cleanly. + """ + without_host = URL.from_text("http:relative-path") + self.assertEqual(without_host.host, "") + self.assertEqual(without_host.path, ("relative-path",)) + self.assertEqual(without_host.uses_netloc, False) + self.assertEqual(without_host.to_text(), "http:relative-path") + + def test_queryIterable(self): + # type: () -> None + """ + When a L{URL} is created with a C{query} argument, the C{query} + argument is converted into an N-tuple of 2-tuples, sensibly + handling dictionaries. + """ + expected = (("alpha", "beta"),) + url = URL(query=[("alpha", "beta")]) + self.assertEqual(url.query, expected) + url = URL(query={"alpha": "beta"}) + self.assertEqual(url.query, expected) + + def test_pathIterable(self): + # type: () -> None + """ + When a L{URL} is created with a C{path} argument, the C{path} is + converted into a tuple. + """ + url = URL(path=["hello", "world"]) + self.assertEqual(url.path, ("hello", "world")) + + def test_invalidArguments(self): + # type: () -> None + """ + Passing an argument of the wrong type to any of the constructor + arguments of L{URL} will raise a descriptive L{TypeError}. + + L{URL} typechecks very aggressively to ensure that its constitutent + parts are all properly immutable and to prevent confusing errors when + bad data crops up in a method call long after the code that called the + constructor is off the stack. + """ + + class Unexpected(object): + def __str__(self): + # type: () -> str + return "wrong" + + def __repr__(self): + # type: () -> str + return "" + + defaultExpectation = "unicode" if bytes is str else "str" + + def assertRaised(raised, expectation, name): + # type: (Any, Text, Text) -> None + self.assertEqual( + str(raised.exception), + "expected {0} for {1}, got {2}".format( + expectation, name, "" + ), + ) + + def check(param, expectation=defaultExpectation): + # type: (Any, str) -> None + with self.assertRaises(TypeError) as raised: + URL(**{param: Unexpected()}) # type: ignore[arg-type] + + assertRaised(raised, expectation, param) + + check("scheme") + check("host") + check("fragment") + check("rooted", "bool") + check("userinfo") + check("port", "int or NoneType") + + with self.assertRaises(TypeError) as raised: + URL(path=[cast(Text, Unexpected())]) + + assertRaised(raised, defaultExpectation, "path segment") + + with self.assertRaises(TypeError) as raised: + URL(query=[("name", cast(Text, Unexpected()))]) + + assertRaised( + raised, defaultExpectation + " or NoneType", "query parameter value" + ) + + with self.assertRaises(TypeError) as raised: + URL(query=[(cast(Text, Unexpected()), "value")]) + + assertRaised(raised, defaultExpectation, "query parameter name") + # No custom error message for this one, just want to make sure + # non-2-tuples don't get through. + + with self.assertRaises(TypeError): + URL(query=[cast(Tuple[Text, Text], Unexpected())]) + + with self.assertRaises(ValueError): + URL(query=[cast(Tuple[Text, Text], ("k", "v", "vv"))]) + + with self.assertRaises(ValueError): + URL(query=[cast(Tuple[Text, Text], ("k",))]) + + url = URL.from_text("https://valid.example.com/") + with self.assertRaises(TypeError) as raised: + url.child(cast(Text, Unexpected())) + assertRaised(raised, defaultExpectation, "path segment") + with self.assertRaises(TypeError) as raised: + url.sibling(cast(Text, Unexpected())) + assertRaised(raised, defaultExpectation, "path segment") + with self.assertRaises(TypeError) as raised: + url.click(cast(Text, Unexpected())) + assertRaised(raised, defaultExpectation, "relative URL") + + def test_technicallyTextIsIterableBut(self): + # type: () -> None + """ + Technically, L{str} (or L{unicode}, as appropriate) is iterable, but + C{URL(path="foo")} resulting in C{URL.from_text("f/o/o")} is never what + you want. + """ + with self.assertRaises(TypeError) as raised: + URL(path="foo") + self.assertEqual( + str(raised.exception), + "expected iterable of text for path, not: {0}".format(repr("foo")), + ) + + def test_netloc(self): + # type: () -> None + url = URL(scheme="https") + self.assertEqual(url.uses_netloc, True) + self.assertEqual(url.to_text(), "https://") + # scheme, no host, no path, no netloc hack + self.assertEqual(URL.from_text("https:").uses_netloc, False) + # scheme, no host, absolute path, no netloc hack + self.assertEqual(URL.from_text("https:/").uses_netloc, False) + # scheme, no host, no path, netloc hack to indicate :// syntax + self.assertEqual(URL.from_text("https://").uses_netloc, True) + + url = URL(scheme="https", uses_netloc=False) + self.assertEqual(url.uses_netloc, False) + self.assertEqual(url.to_text(), "https:") + + url = URL(scheme="git+https") + self.assertEqual(url.uses_netloc, True) + self.assertEqual(url.to_text(), "git+https://") + + url = URL(scheme="mailto") + self.assertEqual(url.uses_netloc, False) + self.assertEqual(url.to_text(), "mailto:") + + url = URL(scheme="ztp") + self.assertEqual(url.uses_netloc, None) + self.assertEqual(url.to_text(), "ztp:") + + url = URL.from_text("ztp://test.com") + self.assertEqual(url.uses_netloc, True) + + url = URL.from_text("ztp:test:com") + self.assertEqual(url.uses_netloc, False) + + def test_ipv6_with_port(self): + # type: () -> None + t = "https://[2001:0db8:85a3:0000:0000:8a2e:0370:7334]:80/" + url = URL.from_text(t) + assert url.host == "2001:0db8:85a3:0000:0000:8a2e:0370:7334" + assert url.port == 80 + assert SCHEME_PORT_MAP[url.scheme] != url.port + + def test_basic(self): + # type: () -> None + text = "https://user:pass@example.com/path/to/here?k=v#nice" + url = URL.from_text(text) + assert url.scheme == "https" + assert url.userinfo == "user:pass" + assert url.host == "example.com" + assert url.path == ("path", "to", "here") + assert url.fragment == "nice" + + text = "https://user:pass@127.0.0.1/path/to/here?k=v#nice" + url = URL.from_text(text) + assert url.scheme == "https" + assert url.userinfo == "user:pass" + assert url.host == "127.0.0.1" + assert url.path == ("path", "to", "here") + + text = "https://user:pass@[::1]/path/to/here?k=v#nice" + url = URL.from_text(text) + assert url.scheme == "https" + assert url.userinfo == "user:pass" + assert url.host == "::1" + assert url.path == ("path", "to", "here") + + def test_invalid_url(self): + # type: () -> None + self.assertRaises(URLParseError, URL.from_text, "#\n\n") + + def test_invalid_authority_url(self): + # type: () -> None + self.assertRaises(URLParseError, URL.from_text, "http://abc:\n\n/#") + + def test_invalid_ipv6(self): + # type: () -> None + invalid_ipv6_ips = [ + "2001::0234:C1ab::A0:aabc:003F", + "2001::1::3F", + ":", + "::::", + "::256.0.0.1", + ] + for ip in invalid_ipv6_ips: + url_text = "http://[" + ip + "]" + self.assertRaises(socket.error, inet_pton, socket.AF_INET6, ip) + self.assertRaises(URLParseError, URL.from_text, url_text) + + def test_invalid_port(self): + # type: () -> None + self.assertRaises(URLParseError, URL.from_text, "ftp://portmouth:smash") + self.assertRaises( + ValueError, + URL.from_text, + "http://reader.googlewebsite.com:neverforget", + ) + + def test_idna(self): + # type: () -> None + u1 = URL.from_text("http://bücher.ch") + self.assertEqual(u1.host, "bücher.ch") + self.assertEqual(u1.to_text(), "http://bücher.ch") + self.assertEqual(u1.to_uri().to_text(), "http://xn--bcher-kva.ch") + + u2 = URL.from_text("https://xn--bcher-kva.ch") + self.assertEqual(u2.host, "xn--bcher-kva.ch") + self.assertEqual(u2.to_text(), "https://xn--bcher-kva.ch") + self.assertEqual(u2.to_iri().to_text(), "https://bücher.ch") + + def test_netloc_slashes(self): + # type: () -> None + + # basic sanity checks + url = URL.from_text("mailto:mahmoud@hatnote.com") + self.assertEqual(url.scheme, "mailto") + self.assertEqual(url.to_text(), "mailto:mahmoud@hatnote.com") + + url = URL.from_text("http://hatnote.com") + self.assertEqual(url.scheme, "http") + self.assertEqual(url.to_text(), "http://hatnote.com") + + # test that unrecognized schemes stay consistent with '//' + url = URL.from_text("newscheme:a:b:c") + self.assertEqual(url.scheme, "newscheme") + self.assertEqual(url.to_text(), "newscheme:a:b:c") + + url = URL.from_text("newerscheme://a/b/c") + self.assertEqual(url.scheme, "newerscheme") + self.assertEqual(url.to_text(), "newerscheme://a/b/c") + + # test that reasonable guesses are made + url = URL.from_text("git+ftp://gitstub.biz/glyph/lefkowitz") + self.assertEqual(url.scheme, "git+ftp") + self.assertEqual(url.to_text(), "git+ftp://gitstub.biz/glyph/lefkowitz") + + url = URL.from_text("what+mailto:freerealestate@enotuniq.org") + self.assertEqual(url.scheme, "what+mailto") + self.assertEqual( + url.to_text(), "what+mailto:freerealestate@enotuniq.org" + ) + + url = URL(scheme="ztp", path=("x", "y", "z"), rooted=True) + self.assertEqual(url.to_text(), "ztp:/x/y/z") + + # also works when the input doesn't include '//' + url = URL( + scheme="git+ftp", + path=("x", "y", "z", ""), + rooted=True, + uses_netloc=True, + ) + # broken bc urlunsplit + self.assertEqual(url.to_text(), "git+ftp:///x/y/z/") + + # really why would this ever come up but ok + url = URL.from_text("file:///path/to/heck") + url2 = url.replace(scheme="mailto") + self.assertEqual(url2.to_text(), "mailto:/path/to/heck") + + url_text = "unregisteredscheme:///a/b/c" + url = URL.from_text(url_text) + no_netloc_url = url.replace(uses_netloc=False) + self.assertEqual(no_netloc_url.to_text(), "unregisteredscheme:/a/b/c") + netloc_url = url.replace(uses_netloc=True) + self.assertEqual(netloc_url.to_text(), url_text) + + return + + def test_rooted_to_relative(self): + # type: () -> None + """ + On host-relative URLs, the C{rooted} flag can be updated to indicate + that the path should no longer be treated as absolute. + """ + a = URL(path=["hello"]) + self.assertEqual(a.to_text(), "hello") + b = a.replace(rooted=True) + self.assertEqual(b.to_text(), "/hello") + self.assertNotEqual(a, b) + + def test_autorooted(self): + # type: () -> None + """ + The C{rooted} flag can be updated in some cases, but it cannot be made + to conflict with other facts surrounding the URL; for example, all URLs + involving an authority (host) are inherently rooted because it is not + syntactically possible to express otherwise; also, once an unrooted URL + gains a path that starts with an empty string, that empty string is + elided and it becomes rooted, because these cases are syntactically + indistinguisable in real URL text. + """ + relative_path_rooted = URL(path=["", "foo"], rooted=False) + self.assertEqual(relative_path_rooted.rooted, True) + relative_flag_rooted = URL(path=["foo"], rooted=True) + self.assertEqual(relative_flag_rooted.rooted, True) + self.assertEqual(relative_path_rooted, relative_flag_rooted) + + attempt_unrooted_absolute = URL(host="foo", path=["bar"], rooted=False) + normal_absolute = URL(host="foo", path=["bar"]) + self.assertEqual(attempt_unrooted_absolute, normal_absolute) + self.assertEqual(normal_absolute.rooted, True) + self.assertEqual(attempt_unrooted_absolute.rooted, True) + + def test_rooted_with_port_but_no_host(self): + # type: () -> None + """ + URLs which include a ``://`` netloc-separator for any reason are + inherently rooted, regardless of the value or presence of the + ``rooted`` constructor argument. + + They may include a netloc-separator because their constructor was + directly invoked with an explicit host or port, or because they were + parsed from a string which included the literal ``://`` separator. + """ + directly_constructed = URL(scheme="udp", port=4900, rooted=False) + directly_constructed_implict = URL(scheme="udp", port=4900) + directly_constructed_rooted = URL(scheme="udp", port=4900, rooted=True) + self.assertEqual(directly_constructed.rooted, True) + self.assertEqual(directly_constructed_implict.rooted, True) + self.assertEqual(directly_constructed_rooted.rooted, True) + parsed = URL.from_text("udp://:4900") + self.assertEqual(str(directly_constructed), str(parsed)) + self.assertEqual(str(directly_constructed_implict), str(parsed)) + self.assertEqual(directly_constructed.asText(), parsed.asText()) + self.assertEqual(directly_constructed, parsed) + self.assertEqual(directly_constructed, directly_constructed_implict) + self.assertEqual(directly_constructed, directly_constructed_rooted) + self.assertEqual(directly_constructed_implict, parsed) + self.assertEqual(directly_constructed_rooted, parsed) + + def test_wrong_constructor(self): + # type: () -> None + with self.assertRaises(ValueError): + # whole URL not allowed + URL(BASIC_URL) + with self.assertRaises(ValueError): + # explicitly bad scheme not allowed + URL("HTTP_____more_like_imHoTTeP") + + def test_encoded_userinfo(self): + # type: () -> None + url = URL.from_text("http://user:pass@example.com") + assert url.userinfo == "user:pass" + url = url.replace(userinfo="us%20her:pass") + iri = url.to_iri() + assert ( + iri.to_text(with_password=True) == "http://us her:pass@example.com" + ) + assert iri.to_text(with_password=False) == "http://us her:@example.com" + assert ( + iri.to_uri().to_text(with_password=True) + == "http://us%20her:pass@example.com" + ) + + def test_hash(self): + # type: () -> None + url_map = {} + url1 = URL.from_text("http://blog.hatnote.com/ask?utm_source=geocity") + assert hash(url1) == hash(url1) # sanity + + url_map[url1] = 1 + + url2 = URL.from_text("http://blog.hatnote.com/ask") + url2 = url2.set("utm_source", "geocity") + + url_map[url2] = 2 + + assert len(url_map) == 1 + assert list(url_map.values()) == [2] + + assert hash(URL()) == hash(URL()) # slightly more sanity + + def test_dir(self): + # type: () -> None + url = URL() + res = dir(url) + + assert len(res) > 15 + # twisted compat + assert "fromText" not in res + assert "asText" not in res + assert "asURI" not in res + assert "asIRI" not in res + + def test_twisted_compat(self): + # type: () -> None + url = URL.fromText("http://example.com/a%20té%C3%A9st") + assert url.asText() == "http://example.com/a%20té%C3%A9st" + assert url.asURI().asText() == "http://example.com/a%20t%C3%A9%C3%A9st" + # TODO: assert url.asIRI().asText() == u'http://example.com/a%20téést' + + def test_set_ordering(self): + # type: () -> None + + # TODO + url = URL.from_text("http://example.com/?a=b&c") + url = url.set("x", "x") + url = url.add("x", "y") + assert url.to_text() == "http://example.com/?a=b&x=x&c&x=y" + # Would expect: + # assert url.to_text() == u'http://example.com/?a=b&c&x=x&x=y' + + def test_schemeless_path(self): + # type: () -> None + "See issue #4" + u1 = URL.from_text("urn%3Aietf%3Awg%3Aoauth%3A2.0%3Aoob") + u2 = URL.from_text(u1.to_text()) + assert u1 == u2 # sanity testing roundtripping + + u3 = URL.from_text(u1.to_iri().to_text()) + assert u1 == u3 + assert u2 == u3 + + # test that colons are ok past the first segment + u4 = URL.from_text("first-segment/urn%3Aietf%3Awg%3Aoauth%3A2.0%3Aoob") + u5 = u4.to_iri() + assert u5.to_text() == "first-segment/urn:ietf:wg:oauth:2.0:oob" + + u6 = URL.from_text(u5.to_text()).to_uri() + assert u5 == u6 # colons stay decoded bc they're not in the first seg + + def test_emoji_domain(self): + # type: () -> None + "See issue #7, affecting only narrow builds (2.6-3.3)" + url = URL.from_text("https://xn--vi8hiv.ws") + iri = url.to_iri() + iri.to_text() + # as long as we don't get ValueErrors, we're good + + def test_delim_in_param(self): + # type: () -> None + "Per issue #6 and #8" + self.assertRaises(ValueError, URL, scheme="http", host="a/c") + self.assertRaises(ValueError, URL, path=("?",)) + self.assertRaises(ValueError, URL, path=("#",)) + self.assertRaises(ValueError, URL, query=(("&", "test"))) + + def test_empty_paths_eq(self): + # type: () -> None + u1 = URL.from_text("http://example.com/") + u2 = URL.from_text("http://example.com") + + assert u1 == u2 + + u1 = URL.from_text("http://example.com") + u2 = URL.from_text("http://example.com") + + assert u1 == u2 + + u1 = URL.from_text("http://example.com") + u2 = URL.from_text("http://example.com/") + + assert u1 == u2 + + u1 = URL.from_text("http://example.com/") + u2 = URL.from_text("http://example.com/") + + assert u1 == u2 + + def test_from_text_type(self): + # type: () -> None + assert URL.from_text("#ok").fragment == "ok" # sanity + self.assertRaises(TypeError, URL.from_text, b"bytes://x.y.z") + self.assertRaises(TypeError, URL.from_text, object()) + + def test_from_text_bad_authority(self): + # type: () -> None + + # bad ipv6 brackets + self.assertRaises(URLParseError, URL.from_text, "http://[::1/") + self.assertRaises(URLParseError, URL.from_text, "http://::1]/") + self.assertRaises(URLParseError, URL.from_text, "http://[[::1]/") + self.assertRaises(URLParseError, URL.from_text, "http://[::1]]/") + + # empty port + self.assertRaises(URLParseError, URL.from_text, "http://127.0.0.1:") + # non-integer port + self.assertRaises(URLParseError, URL.from_text, "http://127.0.0.1:hi") + # extra port colon (makes for an invalid host) + self.assertRaises(URLParseError, URL.from_text, "http://127.0.0.1::80") + + def test_normalize(self): + # type: () -> None + url = URL.from_text("HTTP://Example.com/A%61/./../A%61?B%62=C%63#D%64") + assert url.get("Bb") == [] + assert url.get("B%62") == ["C%63"] + assert len(url.path) == 4 + + # test that most expected normalizations happen + norm_url = url.normalize() + + assert norm_url.scheme == "http" + assert norm_url.host == "example.com" + assert norm_url.path == ("Aa",) + assert norm_url.get("Bb") == ["Cc"] + assert norm_url.fragment == "Dd" + assert norm_url.to_text() == "http://example.com/Aa?Bb=Cc#Dd" + + # test that flags work + noop_norm_url = url.normalize( + scheme=False, host=False, path=False, query=False, fragment=False + ) + assert noop_norm_url == url + + # test that empty paths get at least one slash + slashless_url = URL.from_text("http://example.io") + slashful_url = slashless_url.normalize() + assert slashful_url.to_text() == "http://example.io/" + + # test case normalization for percent encoding + delimited_url = URL.from_text("/a%2fb/cd%3f?k%3d=v%23#test") + norm_delimited_url = delimited_url.normalize() + assert norm_delimited_url.to_text() == "/a%2Fb/cd%3F?k%3D=v%23#test" + + # test invalid percent encoding during normalize + assert ( + URL(path=("", "%te%sts")).normalize(percents=False).to_text() + == "/%te%sts" + ) + assert URL(path=("", "%te%sts")).normalize().to_text() == "/%25te%25sts" + + percenty_url = URL( + scheme="ftp", + path=["%%%", "%a%b"], + query=[("%", "%%")], + fragment="%", + userinfo="%:%", + ) + + assert ( + percenty_url.to_text(with_password=True) + == "ftp://%:%@/%%%/%a%b?%=%%#%" + ) + assert ( + percenty_url.normalize().to_text(with_password=True) + == "ftp://%25:%25@/%25%25%25/%25a%25b?%25=%25%25#%25" + ) + + def test_str(self): + # type: () -> None + + # see also issue #49 + text = "http://example.com/á/y%20a%20y/?b=%25" + url = URL.from_text(text) + assert unicode(url) == text + assert bytes(url) == b"http://example.com/%C3%A1/y%20a%20y/?b=%25" + + if PY2: + assert isinstance(str(url), bytes) + assert isinstance(unicode(url), unicode) + else: + assert isinstance(str(url), unicode) + assert isinstance(bytes(url), bytes) + + def test_idna_corners(self): + # type: () -> None + url = URL.from_text("http://abé.com/") + assert url.to_iri().host == "abé.com" + assert url.to_uri().host == "xn--ab-cja.com" + + url = URL.from_text("http://ドメイン.テスト.co.jp#test") + assert url.to_iri().host == "ドメイン.テスト.co.jp" + assert url.to_uri().host == "xn--eckwd4c7c.xn--zckzah.co.jp" + + assert url.to_uri().get_decoded_url().host == "ドメイン.テスト.co.jp" + + text = "http://Example.com" + assert ( + URL.from_text(text).to_uri().get_decoded_url().host == "example.com" + ) diff --git a/WebCrawler/venv/Lib/site-packages/idna-2.10.dist-info/INSTALLER b/WebCrawler/venv/Lib/site-packages/idna-2.10.dist-info/INSTALLER new file mode 100644 index 0000000..a1b589e --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/idna-2.10.dist-info/INSTALLER @@ -0,0 +1 @@ +pip diff --git a/WebCrawler/venv/Lib/site-packages/idna-2.10.dist-info/LICENSE.rst b/WebCrawler/venv/Lib/site-packages/idna-2.10.dist-info/LICENSE.rst new file mode 100644 index 0000000..63664b8 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/idna-2.10.dist-info/LICENSE.rst @@ -0,0 +1,34 @@ +License +------- + +License: bsd-3-clause + +Copyright (c) 2013-2020, Kim Davies. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +#. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + +#. Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided with + the distribution. + +#. Neither the name of the copyright holder nor the names of the + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +#. THIS SOFTWARE IS PROVIDED BY THE CONTRIBUTORS "AS IS" AND ANY + EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR + CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE + USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH + DAMAGE. diff --git a/WebCrawler/venv/Lib/site-packages/idna-2.10.dist-info/METADATA b/WebCrawler/venv/Lib/site-packages/idna-2.10.dist-info/METADATA new file mode 100644 index 0000000..f73c0ff --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/idna-2.10.dist-info/METADATA @@ -0,0 +1,243 @@ +Metadata-Version: 2.1 +Name: idna +Version: 2.10 +Summary: Internationalized Domain Names in Applications (IDNA) +Home-page: https://github.com/kjd/idna +Author: Kim Davies +Author-email: kim@cynosure.com.au +License: BSD-like +Platform: UNKNOWN +Classifier: Development Status :: 5 - Production/Stable +Classifier: Intended Audience :: Developers +Classifier: Intended Audience :: System Administrators +Classifier: License :: OSI Approved :: BSD License +Classifier: Operating System :: OS Independent +Classifier: Programming Language :: Python +Classifier: Programming Language :: Python :: 2 +Classifier: Programming Language :: Python :: 2.7 +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3.4 +Classifier: Programming Language :: Python :: 3.5 +Classifier: Programming Language :: Python :: 3.6 +Classifier: Programming Language :: Python :: 3.7 +Classifier: Programming Language :: Python :: 3.8 +Classifier: Programming Language :: Python :: Implementation :: CPython +Classifier: Programming Language :: Python :: Implementation :: PyPy +Classifier: Topic :: Internet :: Name Service (DNS) +Classifier: Topic :: Software Development :: Libraries :: Python Modules +Classifier: Topic :: Utilities +Requires-Python: >=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.* + +Internationalized Domain Names in Applications (IDNA) +===================================================== + +Support for the Internationalised Domain Names in Applications +(IDNA) protocol as specified in `RFC 5891 `_. +This is the latest version of the protocol and is sometimes referred to as +“IDNA 2008”. + +This library also provides support for Unicode Technical Standard 46, +`Unicode IDNA Compatibility Processing `_. + +This acts as a suitable replacement for the “encodings.idna” module that +comes with the Python standard library, but only supports the +old, deprecated IDNA specification (`RFC 3490 `_). + +Basic functions are simply executed: + +.. code-block:: pycon + + # Python 3 + >>> import idna + >>> idna.encode('ドメイン.テスト') + b'xn--eckwd4c7c.xn--zckzah' + >>> print(idna.decode('xn--eckwd4c7c.xn--zckzah')) + ドメイン.テスト + + # Python 2 + >>> import idna + >>> idna.encode(u'ドメイン.テスト') + 'xn--eckwd4c7c.xn--zckzah' + >>> print idna.decode('xn--eckwd4c7c.xn--zckzah') + ドメイン.テスト + +Packages +-------- + +The latest tagged release version is published in the PyPI repository: + +.. image:: https://badge.fury.io/py/idna.svg + :target: http://badge.fury.io/py/idna + + +Installation +------------ + +To install this library, you can use pip: + +.. code-block:: bash + + $ pip install idna + +Alternatively, you can install the package using the bundled setup script: + +.. code-block:: bash + + $ python setup.py install + +This library works with Python 2.7 and Python 3.4 or later. + + +Usage +----- + +For typical usage, the ``encode`` and ``decode`` functions will take a domain +name argument and perform a conversion to A-labels or U-labels respectively. + +.. code-block:: pycon + + # Python 3 + >>> import idna + >>> idna.encode('ドメイン.テスト') + b'xn--eckwd4c7c.xn--zckzah' + >>> print(idna.decode('xn--eckwd4c7c.xn--zckzah')) + ドメイン.テスト + +You may use the codec encoding and decoding methods using the +``idna.codec`` module: + +.. code-block:: pycon + + # Python 2 + >>> import idna.codec + >>> print u'домена.испытание'.encode('idna') + xn--80ahd1agd.xn--80akhbyknj4f + >>> print 'xn--80ahd1agd.xn--80akhbyknj4f'.decode('idna') + домена.испытание + +Conversions can be applied at a per-label basis using the ``ulabel`` or ``alabel`` +functions if necessary: + +.. code-block:: pycon + + # Python 2 + >>> idna.alabel(u'测试') + 'xn--0zwm56d' + +Compatibility Mapping (UTS #46) ++++++++++++++++++++++++++++++++ + +As described in `RFC 5895 `_, the IDNA +specification no longer normalizes input from different potential ways a user +may input a domain name. This functionality, known as a “mapping”, is now +considered by the specification to be a local user-interface issue distinct +from IDNA conversion functionality. + +This library provides one such mapping, that was developed by the Unicode +Consortium. Known as `Unicode IDNA Compatibility Processing `_, +it provides for both a regular mapping for typical applications, as well as +a transitional mapping to help migrate from older IDNA 2003 applications. + +For example, “Königsgäßchen” is not a permissible label as *LATIN CAPITAL +LETTER K* is not allowed (nor are capital letters in general). UTS 46 will +convert this into lower case prior to applying the IDNA conversion. + +.. code-block:: pycon + + # Python 3 + >>> import idna + >>> idna.encode(u'Königsgäßchen') + ... + idna.core.InvalidCodepoint: Codepoint U+004B at position 1 of 'Königsgäßchen' not allowed + >>> idna.encode('Königsgäßchen', uts46=True) + b'xn--knigsgchen-b4a3dun' + >>> print(idna.decode('xn--knigsgchen-b4a3dun')) + königsgäßchen + +Transitional processing provides conversions to help transition from the older +2003 standard to the current standard. For example, in the original IDNA +specification, the *LATIN SMALL LETTER SHARP S* (ß) was converted into two +*LATIN SMALL LETTER S* (ss), whereas in the current IDNA specification this +conversion is not performed. + +.. code-block:: pycon + + # Python 2 + >>> idna.encode(u'Königsgäßchen', uts46=True, transitional=True) + 'xn--knigsgsschen-lcb0w' + +Implementors should use transitional processing with caution, only in rare +cases where conversion from legacy labels to current labels must be performed +(i.e. IDNA implementations that pre-date 2008). For typical applications +that just need to convert labels, transitional processing is unlikely to be +beneficial and could produce unexpected incompatible results. + +``encodings.idna`` Compatibility +++++++++++++++++++++++++++++++++ + +Function calls from the Python built-in ``encodings.idna`` module are +mapped to their IDNA 2008 equivalents using the ``idna.compat`` module. +Simply substitute the ``import`` clause in your code to refer to the +new module name. + +Exceptions +---------- + +All errors raised during the conversion following the specification should +raise an exception derived from the ``idna.IDNAError`` base class. + +More specific exceptions that may be generated as ``idna.IDNABidiError`` +when the error reflects an illegal combination of left-to-right and right-to-left +characters in a label; ``idna.InvalidCodepoint`` when a specific codepoint is +an illegal character in an IDN label (i.e. INVALID); and ``idna.InvalidCodepointContext`` +when the codepoint is illegal based on its positional context (i.e. it is CONTEXTO +or CONTEXTJ but the contextual requirements are not satisfied.) + +Building and Diagnostics +------------------------ + +The IDNA and UTS 46 functionality relies upon pre-calculated lookup tables for +performance. These tables are derived from computing against eligibility criteria +in the respective standards. These tables are computed using the command-line +script ``tools/idna-data``. + +This tool will fetch relevant tables from the Unicode Consortium and perform the +required calculations to identify eligibility. It has three main modes: + +* ``idna-data make-libdata``. Generates ``idnadata.py`` and ``uts46data.py``, + the pre-calculated lookup tables using for IDNA and UTS 46 conversions. Implementors + who wish to track this library against a different Unicode version may use this tool + to manually generate a different version of the ``idnadata.py`` and ``uts46data.py`` + files. + +* ``idna-data make-table``. Generate a table of the IDNA disposition + (e.g. PVALID, CONTEXTJ, CONTEXTO) in the format found in Appendix B.1 of RFC + 5892 and the pre-computed tables published by `IANA `_. + +* ``idna-data U+0061``. Prints debugging output on the various properties + associated with an individual Unicode codepoint (in this case, U+0061), that are + used to assess the IDNA and UTS 46 status of a codepoint. This is helpful in debugging + or analysis. + +The tool accepts a number of arguments, described using ``idna-data -h``. Most notably, +the ``--version`` argument allows the specification of the version of Unicode to use +in computing the table data. For example, ``idna-data --version 9.0.0 make-libdata`` +will generate library data against Unicode 9.0.0. + +Note that this script requires Python 3, but all generated library data will work +in Python 2.7. + + +Testing +------- + +The library has a test suite based on each rule of the IDNA specification, as +well as tests that are provided as part of the Unicode Technical Standard 46, +`Unicode IDNA Compatibility Processing `_. + +The tests are run automatically on each commit at Travis CI: + +.. image:: https://travis-ci.org/kjd/idna.svg?branch=master + :target: https://travis-ci.org/kjd/idna + + diff --git a/WebCrawler/venv/Lib/site-packages/idna-2.10.dist-info/RECORD b/WebCrawler/venv/Lib/site-packages/idna-2.10.dist-info/RECORD new file mode 100644 index 0000000..26937cc --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/idna-2.10.dist-info/RECORD @@ -0,0 +1,22 @@ +idna-2.10.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4 +idna-2.10.dist-info/LICENSE.rst,sha256=QSAUQg0kc9ugYRfD1Nng7sqm3eDKMM2VH07CvjlCbzI,1565 +idna-2.10.dist-info/METADATA,sha256=ZWCaQDBjdmSvx5EU7Cv6ORC-9NUQ6nXh1eXx38ySe40,9104 +idna-2.10.dist-info/RECORD,, +idna-2.10.dist-info/WHEEL,sha256=8zNYZbwQSXoB9IfXOjPfeNwvAsALAjffgk27FqvCWbo,110 +idna-2.10.dist-info/top_level.txt,sha256=jSag9sEDqvSPftxOQy-ABfGV_RSy7oFh4zZJpODV8k0,5 +idna/__init__.py,sha256=9Nt7xpyet3DmOrPUGooDdAwmHZZu1qUAy2EaJ93kGiQ,58 +idna/__pycache__/__init__.cpython-39.pyc,, +idna/__pycache__/codec.cpython-39.pyc,, +idna/__pycache__/compat.cpython-39.pyc,, +idna/__pycache__/core.cpython-39.pyc,, +idna/__pycache__/idnadata.cpython-39.pyc,, +idna/__pycache__/intranges.cpython-39.pyc,, +idna/__pycache__/package_data.cpython-39.pyc,, +idna/__pycache__/uts46data.cpython-39.pyc,, +idna/codec.py,sha256=lvYb7yu7PhAqFaAIAdWcwgaWI2UmgseUua-1c0AsG0A,3299 +idna/compat.py,sha256=R-h29D-6mrnJzbXxymrWUW7iZUvy-26TQwZ0ij57i4U,232 +idna/core.py,sha256=jCoaLb3bA2tS_DDx9PpGuNTEZZN2jAzB369aP-IHYRE,11951 +idna/idnadata.py,sha256=gmzFwZWjdms3kKZ_M_vwz7-LP_SCgYfSeE03B21Qpsk,42350 +idna/intranges.py,sha256=TY1lpxZIQWEP6tNqjZkFA5hgoMWOj1OBmnUG8ihT87E,1749 +idna/package_data.py,sha256=bxBjpLnE06_1jSYKEy5svOMu1zM3OMztXVUb1tPlcp0,22 +idna/uts46data.py,sha256=lMdw2zdjkH1JUWXPPEfFUSYT3Fyj60bBmfLvvy5m7ko,202084 diff --git a/WebCrawler/venv/Lib/site-packages/idna-2.10.dist-info/WHEEL b/WebCrawler/venv/Lib/site-packages/idna-2.10.dist-info/WHEEL new file mode 100644 index 0000000..8b701e9 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/idna-2.10.dist-info/WHEEL @@ -0,0 +1,6 @@ +Wheel-Version: 1.0 +Generator: bdist_wheel (0.33.6) +Root-Is-Purelib: true +Tag: py2-none-any +Tag: py3-none-any + diff --git a/WebCrawler/venv/Lib/site-packages/idna-2.10.dist-info/top_level.txt b/WebCrawler/venv/Lib/site-packages/idna-2.10.dist-info/top_level.txt new file mode 100644 index 0000000..c40472e --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/idna-2.10.dist-info/top_level.txt @@ -0,0 +1 @@ +idna diff --git a/WebCrawler/venv/Lib/site-packages/idna/__init__.py b/WebCrawler/venv/Lib/site-packages/idna/__init__.py new file mode 100644 index 0000000..847bf93 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/idna/__init__.py @@ -0,0 +1,2 @@ +from .package_data import __version__ +from .core import * diff --git a/WebCrawler/venv/Lib/site-packages/idna/__pycache__/__init__.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/idna/__pycache__/__init__.cpython-39.pyc new file mode 100644 index 0000000..52e27a1 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/idna/__pycache__/__init__.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/idna/__pycache__/codec.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/idna/__pycache__/codec.cpython-39.pyc new file mode 100644 index 0000000..83c45ab Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/idna/__pycache__/codec.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/idna/__pycache__/compat.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/idna/__pycache__/compat.cpython-39.pyc new file mode 100644 index 0000000..3e82c08 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/idna/__pycache__/compat.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/idna/__pycache__/core.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/idna/__pycache__/core.cpython-39.pyc new file mode 100644 index 0000000..13d7f7b Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/idna/__pycache__/core.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/idna/__pycache__/idnadata.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/idna/__pycache__/idnadata.cpython-39.pyc new file mode 100644 index 0000000..ea3f516 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/idna/__pycache__/idnadata.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/idna/__pycache__/intranges.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/idna/__pycache__/intranges.cpython-39.pyc new file mode 100644 index 0000000..569adb9 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/idna/__pycache__/intranges.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/idna/__pycache__/package_data.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/idna/__pycache__/package_data.cpython-39.pyc new file mode 100644 index 0000000..e90e023 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/idna/__pycache__/package_data.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/idna/__pycache__/uts46data.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/idna/__pycache__/uts46data.cpython-39.pyc new file mode 100644 index 0000000..11c7577 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/idna/__pycache__/uts46data.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/idna/codec.py b/WebCrawler/venv/Lib/site-packages/idna/codec.py new file mode 100644 index 0000000..98c65ea --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/idna/codec.py @@ -0,0 +1,118 @@ +from .core import encode, decode, alabel, ulabel, IDNAError +import codecs +import re + +_unicode_dots_re = re.compile(u'[\u002e\u3002\uff0e\uff61]') + +class Codec(codecs.Codec): + + def encode(self, data, errors='strict'): + + if errors != 'strict': + raise IDNAError("Unsupported error handling \"{0}\"".format(errors)) + + if not data: + return "", 0 + + return encode(data), len(data) + + def decode(self, data, errors='strict'): + + if errors != 'strict': + raise IDNAError("Unsupported error handling \"{0}\"".format(errors)) + + if not data: + return u"", 0 + + return decode(data), len(data) + +class IncrementalEncoder(codecs.BufferedIncrementalEncoder): + def _buffer_encode(self, data, errors, final): + if errors != 'strict': + raise IDNAError("Unsupported error handling \"{0}\"".format(errors)) + + if not data: + return ("", 0) + + labels = _unicode_dots_re.split(data) + trailing_dot = u'' + if labels: + if not labels[-1]: + trailing_dot = '.' + del labels[-1] + elif not final: + # Keep potentially unfinished label until the next call + del labels[-1] + if labels: + trailing_dot = '.' + + result = [] + size = 0 + for label in labels: + result.append(alabel(label)) + if size: + size += 1 + size += len(label) + + # Join with U+002E + result = ".".join(result) + trailing_dot + size += len(trailing_dot) + return (result, size) + +class IncrementalDecoder(codecs.BufferedIncrementalDecoder): + def _buffer_decode(self, data, errors, final): + if errors != 'strict': + raise IDNAError("Unsupported error handling \"{0}\"".format(errors)) + + if not data: + return (u"", 0) + + # IDNA allows decoding to operate on Unicode strings, too. + if isinstance(data, unicode): + labels = _unicode_dots_re.split(data) + else: + # Must be ASCII string + data = str(data) + unicode(data, "ascii") + labels = data.split(".") + + trailing_dot = u'' + if labels: + if not labels[-1]: + trailing_dot = u'.' + del labels[-1] + elif not final: + # Keep potentially unfinished label until the next call + del labels[-1] + if labels: + trailing_dot = u'.' + + result = [] + size = 0 + for label in labels: + result.append(ulabel(label)) + if size: + size += 1 + size += len(label) + + result = u".".join(result) + trailing_dot + size += len(trailing_dot) + return (result, size) + + +class StreamWriter(Codec, codecs.StreamWriter): + pass + +class StreamReader(Codec, codecs.StreamReader): + pass + +def getregentry(): + return codecs.CodecInfo( + name='idna', + encode=Codec().encode, + decode=Codec().decode, + incrementalencoder=IncrementalEncoder, + incrementaldecoder=IncrementalDecoder, + streamwriter=StreamWriter, + streamreader=StreamReader, + ) diff --git a/WebCrawler/venv/Lib/site-packages/idna/compat.py b/WebCrawler/venv/Lib/site-packages/idna/compat.py new file mode 100644 index 0000000..4d47f33 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/idna/compat.py @@ -0,0 +1,12 @@ +from .core import * +from .codec import * + +def ToASCII(label): + return encode(label) + +def ToUnicode(label): + return decode(label) + +def nameprep(s): + raise NotImplementedError("IDNA 2008 does not utilise nameprep protocol") + diff --git a/WebCrawler/venv/Lib/site-packages/idna/core.py b/WebCrawler/venv/Lib/site-packages/idna/core.py new file mode 100644 index 0000000..41ec5c7 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/idna/core.py @@ -0,0 +1,400 @@ +from . import idnadata +import bisect +import unicodedata +import re +import sys +from .intranges import intranges_contain + +_virama_combining_class = 9 +_alabel_prefix = b'xn--' +_unicode_dots_re = re.compile(u'[\u002e\u3002\uff0e\uff61]') + +if sys.version_info[0] >= 3: + unicode = str + unichr = chr + +class IDNAError(UnicodeError): + """ Base exception for all IDNA-encoding related problems """ + pass + + +class IDNABidiError(IDNAError): + """ Exception when bidirectional requirements are not satisfied """ + pass + + +class InvalidCodepoint(IDNAError): + """ Exception when a disallowed or unallocated codepoint is used """ + pass + + +class InvalidCodepointContext(IDNAError): + """ Exception when the codepoint is not valid in the context it is used """ + pass + + +def _combining_class(cp): + v = unicodedata.combining(unichr(cp)) + if v == 0: + if not unicodedata.name(unichr(cp)): + raise ValueError("Unknown character in unicodedata") + return v + +def _is_script(cp, script): + return intranges_contain(ord(cp), idnadata.scripts[script]) + +def _punycode(s): + return s.encode('punycode') + +def _unot(s): + return 'U+{0:04X}'.format(s) + + +def valid_label_length(label): + + if len(label) > 63: + return False + return True + + +def valid_string_length(label, trailing_dot): + + if len(label) > (254 if trailing_dot else 253): + return False + return True + + +def check_bidi(label, check_ltr=False): + + # Bidi rules should only be applied if string contains RTL characters + bidi_label = False + for (idx, cp) in enumerate(label, 1): + direction = unicodedata.bidirectional(cp) + if direction == '': + # String likely comes from a newer version of Unicode + raise IDNABidiError('Unknown directionality in label {0} at position {1}'.format(repr(label), idx)) + if direction in ['R', 'AL', 'AN']: + bidi_label = True + if not bidi_label and not check_ltr: + return True + + # Bidi rule 1 + direction = unicodedata.bidirectional(label[0]) + if direction in ['R', 'AL']: + rtl = True + elif direction == 'L': + rtl = False + else: + raise IDNABidiError('First codepoint in label {0} must be directionality L, R or AL'.format(repr(label))) + + valid_ending = False + number_type = False + for (idx, cp) in enumerate(label, 1): + direction = unicodedata.bidirectional(cp) + + if rtl: + # Bidi rule 2 + if not direction in ['R', 'AL', 'AN', 'EN', 'ES', 'CS', 'ET', 'ON', 'BN', 'NSM']: + raise IDNABidiError('Invalid direction for codepoint at position {0} in a right-to-left label'.format(idx)) + # Bidi rule 3 + if direction in ['R', 'AL', 'EN', 'AN']: + valid_ending = True + elif direction != 'NSM': + valid_ending = False + # Bidi rule 4 + if direction in ['AN', 'EN']: + if not number_type: + number_type = direction + else: + if number_type != direction: + raise IDNABidiError('Can not mix numeral types in a right-to-left label') + else: + # Bidi rule 5 + if not direction in ['L', 'EN', 'ES', 'CS', 'ET', 'ON', 'BN', 'NSM']: + raise IDNABidiError('Invalid direction for codepoint at position {0} in a left-to-right label'.format(idx)) + # Bidi rule 6 + if direction in ['L', 'EN']: + valid_ending = True + elif direction != 'NSM': + valid_ending = False + + if not valid_ending: + raise IDNABidiError('Label ends with illegal codepoint directionality') + + return True + + +def check_initial_combiner(label): + + if unicodedata.category(label[0])[0] == 'M': + raise IDNAError('Label begins with an illegal combining character') + return True + + +def check_hyphen_ok(label): + + if label[2:4] == '--': + raise IDNAError('Label has disallowed hyphens in 3rd and 4th position') + if label[0] == '-' or label[-1] == '-': + raise IDNAError('Label must not start or end with a hyphen') + return True + + +def check_nfc(label): + + if unicodedata.normalize('NFC', label) != label: + raise IDNAError('Label must be in Normalization Form C') + + +def valid_contextj(label, pos): + + cp_value = ord(label[pos]) + + if cp_value == 0x200c: + + if pos > 0: + if _combining_class(ord(label[pos - 1])) == _virama_combining_class: + return True + + ok = False + for i in range(pos-1, -1, -1): + joining_type = idnadata.joining_types.get(ord(label[i])) + if joining_type == ord('T'): + continue + if joining_type in [ord('L'), ord('D')]: + ok = True + break + + if not ok: + return False + + ok = False + for i in range(pos+1, len(label)): + joining_type = idnadata.joining_types.get(ord(label[i])) + if joining_type == ord('T'): + continue + if joining_type in [ord('R'), ord('D')]: + ok = True + break + return ok + + if cp_value == 0x200d: + + if pos > 0: + if _combining_class(ord(label[pos - 1])) == _virama_combining_class: + return True + return False + + else: + + return False + + +def valid_contexto(label, pos, exception=False): + + cp_value = ord(label[pos]) + + if cp_value == 0x00b7: + if 0 < pos < len(label)-1: + if ord(label[pos - 1]) == 0x006c and ord(label[pos + 1]) == 0x006c: + return True + return False + + elif cp_value == 0x0375: + if pos < len(label)-1 and len(label) > 1: + return _is_script(label[pos + 1], 'Greek') + return False + + elif cp_value == 0x05f3 or cp_value == 0x05f4: + if pos > 0: + return _is_script(label[pos - 1], 'Hebrew') + return False + + elif cp_value == 0x30fb: + for cp in label: + if cp == u'\u30fb': + continue + if _is_script(cp, 'Hiragana') or _is_script(cp, 'Katakana') or _is_script(cp, 'Han'): + return True + return False + + elif 0x660 <= cp_value <= 0x669: + for cp in label: + if 0x6f0 <= ord(cp) <= 0x06f9: + return False + return True + + elif 0x6f0 <= cp_value <= 0x6f9: + for cp in label: + if 0x660 <= ord(cp) <= 0x0669: + return False + return True + + +def check_label(label): + + if isinstance(label, (bytes, bytearray)): + label = label.decode('utf-8') + if len(label) == 0: + raise IDNAError('Empty Label') + + check_nfc(label) + check_hyphen_ok(label) + check_initial_combiner(label) + + for (pos, cp) in enumerate(label): + cp_value = ord(cp) + if intranges_contain(cp_value, idnadata.codepoint_classes['PVALID']): + continue + elif intranges_contain(cp_value, idnadata.codepoint_classes['CONTEXTJ']): + try: + if not valid_contextj(label, pos): + raise InvalidCodepointContext('Joiner {0} not allowed at position {1} in {2}'.format( + _unot(cp_value), pos+1, repr(label))) + except ValueError: + raise IDNAError('Unknown codepoint adjacent to joiner {0} at position {1} in {2}'.format( + _unot(cp_value), pos+1, repr(label))) + elif intranges_contain(cp_value, idnadata.codepoint_classes['CONTEXTO']): + if not valid_contexto(label, pos): + raise InvalidCodepointContext('Codepoint {0} not allowed at position {1} in {2}'.format(_unot(cp_value), pos+1, repr(label))) + else: + raise InvalidCodepoint('Codepoint {0} at position {1} of {2} not allowed'.format(_unot(cp_value), pos+1, repr(label))) + + check_bidi(label) + + +def alabel(label): + + try: + label = label.encode('ascii') + ulabel(label) + if not valid_label_length(label): + raise IDNAError('Label too long') + return label + except UnicodeEncodeError: + pass + + if not label: + raise IDNAError('No Input') + + label = unicode(label) + check_label(label) + label = _punycode(label) + label = _alabel_prefix + label + + if not valid_label_length(label): + raise IDNAError('Label too long') + + return label + + +def ulabel(label): + + if not isinstance(label, (bytes, bytearray)): + try: + label = label.encode('ascii') + except UnicodeEncodeError: + check_label(label) + return label + + label = label.lower() + if label.startswith(_alabel_prefix): + label = label[len(_alabel_prefix):] + if not label: + raise IDNAError('Malformed A-label, no Punycode eligible content found') + if label.decode('ascii')[-1] == '-': + raise IDNAError('A-label must not end with a hyphen') + else: + check_label(label) + return label.decode('ascii') + + label = label.decode('punycode') + check_label(label) + return label + + +def uts46_remap(domain, std3_rules=True, transitional=False): + """Re-map the characters in the string according to UTS46 processing.""" + from .uts46data import uts46data + output = u"" + try: + for pos, char in enumerate(domain): + code_point = ord(char) + uts46row = uts46data[code_point if code_point < 256 else + bisect.bisect_left(uts46data, (code_point, "Z")) - 1] + status = uts46row[1] + replacement = uts46row[2] if len(uts46row) == 3 else None + if (status == "V" or + (status == "D" and not transitional) or + (status == "3" and not std3_rules and replacement is None)): + output += char + elif replacement is not None and (status == "M" or + (status == "3" and not std3_rules) or + (status == "D" and transitional)): + output += replacement + elif status != "I": + raise IndexError() + return unicodedata.normalize("NFC", output) + except IndexError: + raise InvalidCodepoint( + "Codepoint {0} not allowed at position {1} in {2}".format( + _unot(code_point), pos + 1, repr(domain))) + + +def encode(s, strict=False, uts46=False, std3_rules=False, transitional=False): + + if isinstance(s, (bytes, bytearray)): + s = s.decode("ascii") + if uts46: + s = uts46_remap(s, std3_rules, transitional) + trailing_dot = False + result = [] + if strict: + labels = s.split('.') + else: + labels = _unicode_dots_re.split(s) + if not labels or labels == ['']: + raise IDNAError('Empty domain') + if labels[-1] == '': + del labels[-1] + trailing_dot = True + for label in labels: + s = alabel(label) + if s: + result.append(s) + else: + raise IDNAError('Empty label') + if trailing_dot: + result.append(b'') + s = b'.'.join(result) + if not valid_string_length(s, trailing_dot): + raise IDNAError('Domain too long') + return s + + +def decode(s, strict=False, uts46=False, std3_rules=False): + + if isinstance(s, (bytes, bytearray)): + s = s.decode("ascii") + if uts46: + s = uts46_remap(s, std3_rules, False) + trailing_dot = False + result = [] + if not strict: + labels = _unicode_dots_re.split(s) + else: + labels = s.split(u'.') + if not labels or labels == ['']: + raise IDNAError('Empty domain') + if not labels[-1]: + del labels[-1] + trailing_dot = True + for label in labels: + s = ulabel(label) + if s: + result.append(s) + else: + raise IDNAError('Empty label') + if trailing_dot: + result.append(u'') + return u'.'.join(result) diff --git a/WebCrawler/venv/Lib/site-packages/idna/idnadata.py b/WebCrawler/venv/Lib/site-packages/idna/idnadata.py new file mode 100644 index 0000000..a284e4c --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/idna/idnadata.py @@ -0,0 +1,2050 @@ +# This file is automatically generated by tools/idna-data + +__version__ = "13.0.0" +scripts = { + 'Greek': ( + 0x37000000374, + 0x37500000378, + 0x37a0000037e, + 0x37f00000380, + 0x38400000385, + 0x38600000387, + 0x3880000038b, + 0x38c0000038d, + 0x38e000003a2, + 0x3a3000003e2, + 0x3f000000400, + 0x1d2600001d2b, + 0x1d5d00001d62, + 0x1d6600001d6b, + 0x1dbf00001dc0, + 0x1f0000001f16, + 0x1f1800001f1e, + 0x1f2000001f46, + 0x1f4800001f4e, + 0x1f5000001f58, + 0x1f5900001f5a, + 0x1f5b00001f5c, + 0x1f5d00001f5e, + 0x1f5f00001f7e, + 0x1f8000001fb5, + 0x1fb600001fc5, + 0x1fc600001fd4, + 0x1fd600001fdc, + 0x1fdd00001ff0, + 0x1ff200001ff5, + 0x1ff600001fff, + 0x212600002127, + 0xab650000ab66, + 0x101400001018f, + 0x101a0000101a1, + 0x1d2000001d246, + ), + 'Han': ( + 0x2e8000002e9a, + 0x2e9b00002ef4, + 0x2f0000002fd6, + 0x300500003006, + 0x300700003008, + 0x30210000302a, + 0x30380000303c, + 0x340000004dc0, + 0x4e0000009ffd, + 0xf9000000fa6e, + 0xfa700000fada, + 0x16ff000016ff2, + 0x200000002a6de, + 0x2a7000002b735, + 0x2b7400002b81e, + 0x2b8200002cea2, + 0x2ceb00002ebe1, + 0x2f8000002fa1e, + 0x300000003134b, + ), + 'Hebrew': ( + 0x591000005c8, + 0x5d0000005eb, + 0x5ef000005f5, + 0xfb1d0000fb37, + 0xfb380000fb3d, + 0xfb3e0000fb3f, + 0xfb400000fb42, + 0xfb430000fb45, + 0xfb460000fb50, + ), + 'Hiragana': ( + 0x304100003097, + 0x309d000030a0, + 0x1b0010001b11f, + 0x1b1500001b153, + 0x1f2000001f201, + ), + 'Katakana': ( + 0x30a1000030fb, + 0x30fd00003100, + 0x31f000003200, + 0x32d0000032ff, + 0x330000003358, + 0xff660000ff70, + 0xff710000ff9e, + 0x1b0000001b001, + 0x1b1640001b168, + ), +} +joining_types = { + 0x600: 85, + 0x601: 85, + 0x602: 85, + 0x603: 85, + 0x604: 85, + 0x605: 85, + 0x608: 85, + 0x60b: 85, + 0x620: 68, + 0x621: 85, + 0x622: 82, + 0x623: 82, + 0x624: 82, + 0x625: 82, + 0x626: 68, + 0x627: 82, + 0x628: 68, + 0x629: 82, + 0x62a: 68, + 0x62b: 68, + 0x62c: 68, + 0x62d: 68, + 0x62e: 68, + 0x62f: 82, + 0x630: 82, + 0x631: 82, + 0x632: 82, + 0x633: 68, + 0x634: 68, + 0x635: 68, + 0x636: 68, + 0x637: 68, + 0x638: 68, + 0x639: 68, + 0x63a: 68, + 0x63b: 68, + 0x63c: 68, + 0x63d: 68, + 0x63e: 68, + 0x63f: 68, + 0x640: 67, + 0x641: 68, + 0x642: 68, + 0x643: 68, + 0x644: 68, + 0x645: 68, + 0x646: 68, + 0x647: 68, + 0x648: 82, + 0x649: 68, + 0x64a: 68, + 0x66e: 68, + 0x66f: 68, + 0x671: 82, + 0x672: 82, + 0x673: 82, + 0x674: 85, + 0x675: 82, + 0x676: 82, + 0x677: 82, + 0x678: 68, + 0x679: 68, + 0x67a: 68, + 0x67b: 68, + 0x67c: 68, + 0x67d: 68, + 0x67e: 68, + 0x67f: 68, + 0x680: 68, + 0x681: 68, + 0x682: 68, + 0x683: 68, + 0x684: 68, + 0x685: 68, + 0x686: 68, + 0x687: 68, + 0x688: 82, + 0x689: 82, + 0x68a: 82, + 0x68b: 82, + 0x68c: 82, + 0x68d: 82, + 0x68e: 82, + 0x68f: 82, + 0x690: 82, + 0x691: 82, + 0x692: 82, + 0x693: 82, + 0x694: 82, + 0x695: 82, + 0x696: 82, + 0x697: 82, + 0x698: 82, + 0x699: 82, + 0x69a: 68, + 0x69b: 68, + 0x69c: 68, + 0x69d: 68, + 0x69e: 68, + 0x69f: 68, + 0x6a0: 68, + 0x6a1: 68, + 0x6a2: 68, + 0x6a3: 68, + 0x6a4: 68, + 0x6a5: 68, + 0x6a6: 68, + 0x6a7: 68, + 0x6a8: 68, + 0x6a9: 68, + 0x6aa: 68, + 0x6ab: 68, + 0x6ac: 68, + 0x6ad: 68, + 0x6ae: 68, + 0x6af: 68, + 0x6b0: 68, + 0x6b1: 68, + 0x6b2: 68, + 0x6b3: 68, + 0x6b4: 68, + 0x6b5: 68, + 0x6b6: 68, + 0x6b7: 68, + 0x6b8: 68, + 0x6b9: 68, + 0x6ba: 68, + 0x6bb: 68, + 0x6bc: 68, + 0x6bd: 68, + 0x6be: 68, + 0x6bf: 68, + 0x6c0: 82, + 0x6c1: 68, + 0x6c2: 68, + 0x6c3: 82, + 0x6c4: 82, + 0x6c5: 82, + 0x6c6: 82, + 0x6c7: 82, + 0x6c8: 82, + 0x6c9: 82, + 0x6ca: 82, + 0x6cb: 82, + 0x6cc: 68, + 0x6cd: 82, + 0x6ce: 68, + 0x6cf: 82, + 0x6d0: 68, + 0x6d1: 68, + 0x6d2: 82, + 0x6d3: 82, + 0x6d5: 82, + 0x6dd: 85, + 0x6ee: 82, + 0x6ef: 82, + 0x6fa: 68, + 0x6fb: 68, + 0x6fc: 68, + 0x6ff: 68, + 0x70f: 84, + 0x710: 82, + 0x712: 68, + 0x713: 68, + 0x714: 68, + 0x715: 82, + 0x716: 82, + 0x717: 82, + 0x718: 82, + 0x719: 82, + 0x71a: 68, + 0x71b: 68, + 0x71c: 68, + 0x71d: 68, + 0x71e: 82, + 0x71f: 68, + 0x720: 68, + 0x721: 68, + 0x722: 68, + 0x723: 68, + 0x724: 68, + 0x725: 68, + 0x726: 68, + 0x727: 68, + 0x728: 82, + 0x729: 68, + 0x72a: 82, + 0x72b: 68, + 0x72c: 82, + 0x72d: 68, + 0x72e: 68, + 0x72f: 82, + 0x74d: 82, + 0x74e: 68, + 0x74f: 68, + 0x750: 68, + 0x751: 68, + 0x752: 68, + 0x753: 68, + 0x754: 68, + 0x755: 68, + 0x756: 68, + 0x757: 68, + 0x758: 68, + 0x759: 82, + 0x75a: 82, + 0x75b: 82, + 0x75c: 68, + 0x75d: 68, + 0x75e: 68, + 0x75f: 68, + 0x760: 68, + 0x761: 68, + 0x762: 68, + 0x763: 68, + 0x764: 68, + 0x765: 68, + 0x766: 68, + 0x767: 68, + 0x768: 68, + 0x769: 68, + 0x76a: 68, + 0x76b: 82, + 0x76c: 82, + 0x76d: 68, + 0x76e: 68, + 0x76f: 68, + 0x770: 68, + 0x771: 82, + 0x772: 68, + 0x773: 82, + 0x774: 82, + 0x775: 68, + 0x776: 68, + 0x777: 68, + 0x778: 82, + 0x779: 82, + 0x77a: 68, + 0x77b: 68, + 0x77c: 68, + 0x77d: 68, + 0x77e: 68, + 0x77f: 68, + 0x7ca: 68, + 0x7cb: 68, + 0x7cc: 68, + 0x7cd: 68, + 0x7ce: 68, + 0x7cf: 68, + 0x7d0: 68, + 0x7d1: 68, + 0x7d2: 68, + 0x7d3: 68, + 0x7d4: 68, + 0x7d5: 68, + 0x7d6: 68, + 0x7d7: 68, + 0x7d8: 68, + 0x7d9: 68, + 0x7da: 68, + 0x7db: 68, + 0x7dc: 68, + 0x7dd: 68, + 0x7de: 68, + 0x7df: 68, + 0x7e0: 68, + 0x7e1: 68, + 0x7e2: 68, + 0x7e3: 68, + 0x7e4: 68, + 0x7e5: 68, + 0x7e6: 68, + 0x7e7: 68, + 0x7e8: 68, + 0x7e9: 68, + 0x7ea: 68, + 0x7fa: 67, + 0x840: 82, + 0x841: 68, + 0x842: 68, + 0x843: 68, + 0x844: 68, + 0x845: 68, + 0x846: 82, + 0x847: 82, + 0x848: 68, + 0x849: 82, + 0x84a: 68, + 0x84b: 68, + 0x84c: 68, + 0x84d: 68, + 0x84e: 68, + 0x84f: 68, + 0x850: 68, + 0x851: 68, + 0x852: 68, + 0x853: 68, + 0x854: 82, + 0x855: 68, + 0x856: 82, + 0x857: 82, + 0x858: 82, + 0x860: 68, + 0x861: 85, + 0x862: 68, + 0x863: 68, + 0x864: 68, + 0x865: 68, + 0x866: 85, + 0x867: 82, + 0x868: 68, + 0x869: 82, + 0x86a: 82, + 0x8a0: 68, + 0x8a1: 68, + 0x8a2: 68, + 0x8a3: 68, + 0x8a4: 68, + 0x8a5: 68, + 0x8a6: 68, + 0x8a7: 68, + 0x8a8: 68, + 0x8a9: 68, + 0x8aa: 82, + 0x8ab: 82, + 0x8ac: 82, + 0x8ad: 85, + 0x8ae: 82, + 0x8af: 68, + 0x8b0: 68, + 0x8b1: 82, + 0x8b2: 82, + 0x8b3: 68, + 0x8b4: 68, + 0x8b6: 68, + 0x8b7: 68, + 0x8b8: 68, + 0x8b9: 82, + 0x8ba: 68, + 0x8bb: 68, + 0x8bc: 68, + 0x8bd: 68, + 0x8be: 68, + 0x8bf: 68, + 0x8c0: 68, + 0x8c1: 68, + 0x8c2: 68, + 0x8c3: 68, + 0x8c4: 68, + 0x8c5: 68, + 0x8c6: 68, + 0x8c7: 68, + 0x8e2: 85, + 0x1806: 85, + 0x1807: 68, + 0x180a: 67, + 0x180e: 85, + 0x1820: 68, + 0x1821: 68, + 0x1822: 68, + 0x1823: 68, + 0x1824: 68, + 0x1825: 68, + 0x1826: 68, + 0x1827: 68, + 0x1828: 68, + 0x1829: 68, + 0x182a: 68, + 0x182b: 68, + 0x182c: 68, + 0x182d: 68, + 0x182e: 68, + 0x182f: 68, + 0x1830: 68, + 0x1831: 68, + 0x1832: 68, + 0x1833: 68, + 0x1834: 68, + 0x1835: 68, + 0x1836: 68, + 0x1837: 68, + 0x1838: 68, + 0x1839: 68, + 0x183a: 68, + 0x183b: 68, + 0x183c: 68, + 0x183d: 68, + 0x183e: 68, + 0x183f: 68, + 0x1840: 68, + 0x1841: 68, + 0x1842: 68, + 0x1843: 68, + 0x1844: 68, + 0x1845: 68, + 0x1846: 68, + 0x1847: 68, + 0x1848: 68, + 0x1849: 68, + 0x184a: 68, + 0x184b: 68, + 0x184c: 68, + 0x184d: 68, + 0x184e: 68, + 0x184f: 68, + 0x1850: 68, + 0x1851: 68, + 0x1852: 68, + 0x1853: 68, + 0x1854: 68, + 0x1855: 68, + 0x1856: 68, + 0x1857: 68, + 0x1858: 68, + 0x1859: 68, + 0x185a: 68, + 0x185b: 68, + 0x185c: 68, + 0x185d: 68, + 0x185e: 68, + 0x185f: 68, + 0x1860: 68, + 0x1861: 68, + 0x1862: 68, + 0x1863: 68, + 0x1864: 68, + 0x1865: 68, + 0x1866: 68, + 0x1867: 68, + 0x1868: 68, + 0x1869: 68, + 0x186a: 68, + 0x186b: 68, + 0x186c: 68, + 0x186d: 68, + 0x186e: 68, + 0x186f: 68, + 0x1870: 68, + 0x1871: 68, + 0x1872: 68, + 0x1873: 68, + 0x1874: 68, + 0x1875: 68, + 0x1876: 68, + 0x1877: 68, + 0x1878: 68, + 0x1880: 85, + 0x1881: 85, + 0x1882: 85, + 0x1883: 85, + 0x1884: 85, + 0x1885: 84, + 0x1886: 84, + 0x1887: 68, + 0x1888: 68, + 0x1889: 68, + 0x188a: 68, + 0x188b: 68, + 0x188c: 68, + 0x188d: 68, + 0x188e: 68, + 0x188f: 68, + 0x1890: 68, + 0x1891: 68, + 0x1892: 68, + 0x1893: 68, + 0x1894: 68, + 0x1895: 68, + 0x1896: 68, + 0x1897: 68, + 0x1898: 68, + 0x1899: 68, + 0x189a: 68, + 0x189b: 68, + 0x189c: 68, + 0x189d: 68, + 0x189e: 68, + 0x189f: 68, + 0x18a0: 68, + 0x18a1: 68, + 0x18a2: 68, + 0x18a3: 68, + 0x18a4: 68, + 0x18a5: 68, + 0x18a6: 68, + 0x18a7: 68, + 0x18a8: 68, + 0x18aa: 68, + 0x200c: 85, + 0x200d: 67, + 0x202f: 85, + 0x2066: 85, + 0x2067: 85, + 0x2068: 85, + 0x2069: 85, + 0xa840: 68, + 0xa841: 68, + 0xa842: 68, + 0xa843: 68, + 0xa844: 68, + 0xa845: 68, + 0xa846: 68, + 0xa847: 68, + 0xa848: 68, + 0xa849: 68, + 0xa84a: 68, + 0xa84b: 68, + 0xa84c: 68, + 0xa84d: 68, + 0xa84e: 68, + 0xa84f: 68, + 0xa850: 68, + 0xa851: 68, + 0xa852: 68, + 0xa853: 68, + 0xa854: 68, + 0xa855: 68, + 0xa856: 68, + 0xa857: 68, + 0xa858: 68, + 0xa859: 68, + 0xa85a: 68, + 0xa85b: 68, + 0xa85c: 68, + 0xa85d: 68, + 0xa85e: 68, + 0xa85f: 68, + 0xa860: 68, + 0xa861: 68, + 0xa862: 68, + 0xa863: 68, + 0xa864: 68, + 0xa865: 68, + 0xa866: 68, + 0xa867: 68, + 0xa868: 68, + 0xa869: 68, + 0xa86a: 68, + 0xa86b: 68, + 0xa86c: 68, + 0xa86d: 68, + 0xa86e: 68, + 0xa86f: 68, + 0xa870: 68, + 0xa871: 68, + 0xa872: 76, + 0xa873: 85, + 0x10ac0: 68, + 0x10ac1: 68, + 0x10ac2: 68, + 0x10ac3: 68, + 0x10ac4: 68, + 0x10ac5: 82, + 0x10ac6: 85, + 0x10ac7: 82, + 0x10ac8: 85, + 0x10ac9: 82, + 0x10aca: 82, + 0x10acb: 85, + 0x10acc: 85, + 0x10acd: 76, + 0x10ace: 82, + 0x10acf: 82, + 0x10ad0: 82, + 0x10ad1: 82, + 0x10ad2: 82, + 0x10ad3: 68, + 0x10ad4: 68, + 0x10ad5: 68, + 0x10ad6: 68, + 0x10ad7: 76, + 0x10ad8: 68, + 0x10ad9: 68, + 0x10ada: 68, + 0x10adb: 68, + 0x10adc: 68, + 0x10add: 82, + 0x10ade: 68, + 0x10adf: 68, + 0x10ae0: 68, + 0x10ae1: 82, + 0x10ae2: 85, + 0x10ae3: 85, + 0x10ae4: 82, + 0x10aeb: 68, + 0x10aec: 68, + 0x10aed: 68, + 0x10aee: 68, + 0x10aef: 82, + 0x10b80: 68, + 0x10b81: 82, + 0x10b82: 68, + 0x10b83: 82, + 0x10b84: 82, + 0x10b85: 82, + 0x10b86: 68, + 0x10b87: 68, + 0x10b88: 68, + 0x10b89: 82, + 0x10b8a: 68, + 0x10b8b: 68, + 0x10b8c: 82, + 0x10b8d: 68, + 0x10b8e: 82, + 0x10b8f: 82, + 0x10b90: 68, + 0x10b91: 82, + 0x10ba9: 82, + 0x10baa: 82, + 0x10bab: 82, + 0x10bac: 82, + 0x10bad: 68, + 0x10bae: 68, + 0x10baf: 85, + 0x10d00: 76, + 0x10d01: 68, + 0x10d02: 68, + 0x10d03: 68, + 0x10d04: 68, + 0x10d05: 68, + 0x10d06: 68, + 0x10d07: 68, + 0x10d08: 68, + 0x10d09: 68, + 0x10d0a: 68, + 0x10d0b: 68, + 0x10d0c: 68, + 0x10d0d: 68, + 0x10d0e: 68, + 0x10d0f: 68, + 0x10d10: 68, + 0x10d11: 68, + 0x10d12: 68, + 0x10d13: 68, + 0x10d14: 68, + 0x10d15: 68, + 0x10d16: 68, + 0x10d17: 68, + 0x10d18: 68, + 0x10d19: 68, + 0x10d1a: 68, + 0x10d1b: 68, + 0x10d1c: 68, + 0x10d1d: 68, + 0x10d1e: 68, + 0x10d1f: 68, + 0x10d20: 68, + 0x10d21: 68, + 0x10d22: 82, + 0x10d23: 68, + 0x10f30: 68, + 0x10f31: 68, + 0x10f32: 68, + 0x10f33: 82, + 0x10f34: 68, + 0x10f35: 68, + 0x10f36: 68, + 0x10f37: 68, + 0x10f38: 68, + 0x10f39: 68, + 0x10f3a: 68, + 0x10f3b: 68, + 0x10f3c: 68, + 0x10f3d: 68, + 0x10f3e: 68, + 0x10f3f: 68, + 0x10f40: 68, + 0x10f41: 68, + 0x10f42: 68, + 0x10f43: 68, + 0x10f44: 68, + 0x10f45: 85, + 0x10f51: 68, + 0x10f52: 68, + 0x10f53: 68, + 0x10f54: 82, + 0x10fb0: 68, + 0x10fb1: 85, + 0x10fb2: 68, + 0x10fb3: 68, + 0x10fb4: 82, + 0x10fb5: 82, + 0x10fb6: 82, + 0x10fb7: 85, + 0x10fb8: 68, + 0x10fb9: 82, + 0x10fba: 82, + 0x10fbb: 68, + 0x10fbc: 68, + 0x10fbd: 82, + 0x10fbe: 68, + 0x10fbf: 68, + 0x10fc0: 85, + 0x10fc1: 68, + 0x10fc2: 82, + 0x10fc3: 82, + 0x10fc4: 68, + 0x10fc5: 85, + 0x10fc6: 85, + 0x10fc7: 85, + 0x10fc8: 85, + 0x10fc9: 82, + 0x10fca: 68, + 0x10fcb: 76, + 0x110bd: 85, + 0x110cd: 85, + 0x1e900: 68, + 0x1e901: 68, + 0x1e902: 68, + 0x1e903: 68, + 0x1e904: 68, + 0x1e905: 68, + 0x1e906: 68, + 0x1e907: 68, + 0x1e908: 68, + 0x1e909: 68, + 0x1e90a: 68, + 0x1e90b: 68, + 0x1e90c: 68, + 0x1e90d: 68, + 0x1e90e: 68, + 0x1e90f: 68, + 0x1e910: 68, + 0x1e911: 68, + 0x1e912: 68, + 0x1e913: 68, + 0x1e914: 68, + 0x1e915: 68, + 0x1e916: 68, + 0x1e917: 68, + 0x1e918: 68, + 0x1e919: 68, + 0x1e91a: 68, + 0x1e91b: 68, + 0x1e91c: 68, + 0x1e91d: 68, + 0x1e91e: 68, + 0x1e91f: 68, + 0x1e920: 68, + 0x1e921: 68, + 0x1e922: 68, + 0x1e923: 68, + 0x1e924: 68, + 0x1e925: 68, + 0x1e926: 68, + 0x1e927: 68, + 0x1e928: 68, + 0x1e929: 68, + 0x1e92a: 68, + 0x1e92b: 68, + 0x1e92c: 68, + 0x1e92d: 68, + 0x1e92e: 68, + 0x1e92f: 68, + 0x1e930: 68, + 0x1e931: 68, + 0x1e932: 68, + 0x1e933: 68, + 0x1e934: 68, + 0x1e935: 68, + 0x1e936: 68, + 0x1e937: 68, + 0x1e938: 68, + 0x1e939: 68, + 0x1e93a: 68, + 0x1e93b: 68, + 0x1e93c: 68, + 0x1e93d: 68, + 0x1e93e: 68, + 0x1e93f: 68, + 0x1e940: 68, + 0x1e941: 68, + 0x1e942: 68, + 0x1e943: 68, + 0x1e94b: 84, +} +codepoint_classes = { + 'PVALID': ( + 0x2d0000002e, + 0x300000003a, + 0x610000007b, + 0xdf000000f7, + 0xf800000100, + 0x10100000102, + 0x10300000104, + 0x10500000106, + 0x10700000108, + 0x1090000010a, + 0x10b0000010c, + 0x10d0000010e, + 0x10f00000110, + 0x11100000112, + 0x11300000114, + 0x11500000116, + 0x11700000118, + 0x1190000011a, + 0x11b0000011c, + 0x11d0000011e, + 0x11f00000120, + 0x12100000122, + 0x12300000124, + 0x12500000126, + 0x12700000128, + 0x1290000012a, + 0x12b0000012c, + 0x12d0000012e, + 0x12f00000130, + 0x13100000132, + 0x13500000136, + 0x13700000139, + 0x13a0000013b, + 0x13c0000013d, + 0x13e0000013f, + 0x14200000143, + 0x14400000145, + 0x14600000147, + 0x14800000149, + 0x14b0000014c, + 0x14d0000014e, + 0x14f00000150, + 0x15100000152, + 0x15300000154, + 0x15500000156, + 0x15700000158, + 0x1590000015a, + 0x15b0000015c, + 0x15d0000015e, + 0x15f00000160, + 0x16100000162, + 0x16300000164, + 0x16500000166, + 0x16700000168, + 0x1690000016a, + 0x16b0000016c, + 0x16d0000016e, + 0x16f00000170, + 0x17100000172, + 0x17300000174, + 0x17500000176, + 0x17700000178, + 0x17a0000017b, + 0x17c0000017d, + 0x17e0000017f, + 0x18000000181, + 0x18300000184, + 0x18500000186, + 0x18800000189, + 0x18c0000018e, + 0x19200000193, + 0x19500000196, + 0x1990000019c, + 0x19e0000019f, + 0x1a1000001a2, + 0x1a3000001a4, + 0x1a5000001a6, + 0x1a8000001a9, + 0x1aa000001ac, + 0x1ad000001ae, + 0x1b0000001b1, + 0x1b4000001b5, + 0x1b6000001b7, + 0x1b9000001bc, + 0x1bd000001c4, + 0x1ce000001cf, + 0x1d0000001d1, + 0x1d2000001d3, + 0x1d4000001d5, + 0x1d6000001d7, + 0x1d8000001d9, + 0x1da000001db, + 0x1dc000001de, + 0x1df000001e0, + 0x1e1000001e2, + 0x1e3000001e4, + 0x1e5000001e6, + 0x1e7000001e8, + 0x1e9000001ea, + 0x1eb000001ec, + 0x1ed000001ee, + 0x1ef000001f1, + 0x1f5000001f6, + 0x1f9000001fa, + 0x1fb000001fc, + 0x1fd000001fe, + 0x1ff00000200, + 0x20100000202, + 0x20300000204, + 0x20500000206, + 0x20700000208, + 0x2090000020a, + 0x20b0000020c, + 0x20d0000020e, + 0x20f00000210, + 0x21100000212, + 0x21300000214, + 0x21500000216, + 0x21700000218, + 0x2190000021a, + 0x21b0000021c, + 0x21d0000021e, + 0x21f00000220, + 0x22100000222, + 0x22300000224, + 0x22500000226, + 0x22700000228, + 0x2290000022a, + 0x22b0000022c, + 0x22d0000022e, + 0x22f00000230, + 0x23100000232, + 0x2330000023a, + 0x23c0000023d, + 0x23f00000241, + 0x24200000243, + 0x24700000248, + 0x2490000024a, + 0x24b0000024c, + 0x24d0000024e, + 0x24f000002b0, + 0x2b9000002c2, + 0x2c6000002d2, + 0x2ec000002ed, + 0x2ee000002ef, + 0x30000000340, + 0x34200000343, + 0x3460000034f, + 0x35000000370, + 0x37100000372, + 0x37300000374, + 0x37700000378, + 0x37b0000037e, + 0x39000000391, + 0x3ac000003cf, + 0x3d7000003d8, + 0x3d9000003da, + 0x3db000003dc, + 0x3dd000003de, + 0x3df000003e0, + 0x3e1000003e2, + 0x3e3000003e4, + 0x3e5000003e6, + 0x3e7000003e8, + 0x3e9000003ea, + 0x3eb000003ec, + 0x3ed000003ee, + 0x3ef000003f0, + 0x3f3000003f4, + 0x3f8000003f9, + 0x3fb000003fd, + 0x43000000460, + 0x46100000462, + 0x46300000464, + 0x46500000466, + 0x46700000468, + 0x4690000046a, + 0x46b0000046c, + 0x46d0000046e, + 0x46f00000470, + 0x47100000472, + 0x47300000474, + 0x47500000476, + 0x47700000478, + 0x4790000047a, + 0x47b0000047c, + 0x47d0000047e, + 0x47f00000480, + 0x48100000482, + 0x48300000488, + 0x48b0000048c, + 0x48d0000048e, + 0x48f00000490, + 0x49100000492, + 0x49300000494, + 0x49500000496, + 0x49700000498, + 0x4990000049a, + 0x49b0000049c, + 0x49d0000049e, + 0x49f000004a0, + 0x4a1000004a2, + 0x4a3000004a4, + 0x4a5000004a6, + 0x4a7000004a8, + 0x4a9000004aa, + 0x4ab000004ac, + 0x4ad000004ae, + 0x4af000004b0, + 0x4b1000004b2, + 0x4b3000004b4, + 0x4b5000004b6, + 0x4b7000004b8, + 0x4b9000004ba, + 0x4bb000004bc, + 0x4bd000004be, + 0x4bf000004c0, + 0x4c2000004c3, + 0x4c4000004c5, + 0x4c6000004c7, + 0x4c8000004c9, + 0x4ca000004cb, + 0x4cc000004cd, + 0x4ce000004d0, + 0x4d1000004d2, + 0x4d3000004d4, + 0x4d5000004d6, + 0x4d7000004d8, + 0x4d9000004da, + 0x4db000004dc, + 0x4dd000004de, + 0x4df000004e0, + 0x4e1000004e2, + 0x4e3000004e4, + 0x4e5000004e6, + 0x4e7000004e8, + 0x4e9000004ea, + 0x4eb000004ec, + 0x4ed000004ee, + 0x4ef000004f0, + 0x4f1000004f2, + 0x4f3000004f4, + 0x4f5000004f6, + 0x4f7000004f8, + 0x4f9000004fa, + 0x4fb000004fc, + 0x4fd000004fe, + 0x4ff00000500, + 0x50100000502, + 0x50300000504, + 0x50500000506, + 0x50700000508, + 0x5090000050a, + 0x50b0000050c, + 0x50d0000050e, + 0x50f00000510, + 0x51100000512, + 0x51300000514, + 0x51500000516, + 0x51700000518, + 0x5190000051a, + 0x51b0000051c, + 0x51d0000051e, + 0x51f00000520, + 0x52100000522, + 0x52300000524, + 0x52500000526, + 0x52700000528, + 0x5290000052a, + 0x52b0000052c, + 0x52d0000052e, + 0x52f00000530, + 0x5590000055a, + 0x56000000587, + 0x58800000589, + 0x591000005be, + 0x5bf000005c0, + 0x5c1000005c3, + 0x5c4000005c6, + 0x5c7000005c8, + 0x5d0000005eb, + 0x5ef000005f3, + 0x6100000061b, + 0x62000000640, + 0x64100000660, + 0x66e00000675, + 0x679000006d4, + 0x6d5000006dd, + 0x6df000006e9, + 0x6ea000006f0, + 0x6fa00000700, + 0x7100000074b, + 0x74d000007b2, + 0x7c0000007f6, + 0x7fd000007fe, + 0x8000000082e, + 0x8400000085c, + 0x8600000086b, + 0x8a0000008b5, + 0x8b6000008c8, + 0x8d3000008e2, + 0x8e300000958, + 0x96000000964, + 0x96600000970, + 0x97100000984, + 0x9850000098d, + 0x98f00000991, + 0x993000009a9, + 0x9aa000009b1, + 0x9b2000009b3, + 0x9b6000009ba, + 0x9bc000009c5, + 0x9c7000009c9, + 0x9cb000009cf, + 0x9d7000009d8, + 0x9e0000009e4, + 0x9e6000009f2, + 0x9fc000009fd, + 0x9fe000009ff, + 0xa0100000a04, + 0xa0500000a0b, + 0xa0f00000a11, + 0xa1300000a29, + 0xa2a00000a31, + 0xa3200000a33, + 0xa3500000a36, + 0xa3800000a3a, + 0xa3c00000a3d, + 0xa3e00000a43, + 0xa4700000a49, + 0xa4b00000a4e, + 0xa5100000a52, + 0xa5c00000a5d, + 0xa6600000a76, + 0xa8100000a84, + 0xa8500000a8e, + 0xa8f00000a92, + 0xa9300000aa9, + 0xaaa00000ab1, + 0xab200000ab4, + 0xab500000aba, + 0xabc00000ac6, + 0xac700000aca, + 0xacb00000ace, + 0xad000000ad1, + 0xae000000ae4, + 0xae600000af0, + 0xaf900000b00, + 0xb0100000b04, + 0xb0500000b0d, + 0xb0f00000b11, + 0xb1300000b29, + 0xb2a00000b31, + 0xb3200000b34, + 0xb3500000b3a, + 0xb3c00000b45, + 0xb4700000b49, + 0xb4b00000b4e, + 0xb5500000b58, + 0xb5f00000b64, + 0xb6600000b70, + 0xb7100000b72, + 0xb8200000b84, + 0xb8500000b8b, + 0xb8e00000b91, + 0xb9200000b96, + 0xb9900000b9b, + 0xb9c00000b9d, + 0xb9e00000ba0, + 0xba300000ba5, + 0xba800000bab, + 0xbae00000bba, + 0xbbe00000bc3, + 0xbc600000bc9, + 0xbca00000bce, + 0xbd000000bd1, + 0xbd700000bd8, + 0xbe600000bf0, + 0xc0000000c0d, + 0xc0e00000c11, + 0xc1200000c29, + 0xc2a00000c3a, + 0xc3d00000c45, + 0xc4600000c49, + 0xc4a00000c4e, + 0xc5500000c57, + 0xc5800000c5b, + 0xc6000000c64, + 0xc6600000c70, + 0xc8000000c84, + 0xc8500000c8d, + 0xc8e00000c91, + 0xc9200000ca9, + 0xcaa00000cb4, + 0xcb500000cba, + 0xcbc00000cc5, + 0xcc600000cc9, + 0xcca00000cce, + 0xcd500000cd7, + 0xcde00000cdf, + 0xce000000ce4, + 0xce600000cf0, + 0xcf100000cf3, + 0xd0000000d0d, + 0xd0e00000d11, + 0xd1200000d45, + 0xd4600000d49, + 0xd4a00000d4f, + 0xd5400000d58, + 0xd5f00000d64, + 0xd6600000d70, + 0xd7a00000d80, + 0xd8100000d84, + 0xd8500000d97, + 0xd9a00000db2, + 0xdb300000dbc, + 0xdbd00000dbe, + 0xdc000000dc7, + 0xdca00000dcb, + 0xdcf00000dd5, + 0xdd600000dd7, + 0xdd800000de0, + 0xde600000df0, + 0xdf200000df4, + 0xe0100000e33, + 0xe3400000e3b, + 0xe4000000e4f, + 0xe5000000e5a, + 0xe8100000e83, + 0xe8400000e85, + 0xe8600000e8b, + 0xe8c00000ea4, + 0xea500000ea6, + 0xea700000eb3, + 0xeb400000ebe, + 0xec000000ec5, + 0xec600000ec7, + 0xec800000ece, + 0xed000000eda, + 0xede00000ee0, + 0xf0000000f01, + 0xf0b00000f0c, + 0xf1800000f1a, + 0xf2000000f2a, + 0xf3500000f36, + 0xf3700000f38, + 0xf3900000f3a, + 0xf3e00000f43, + 0xf4400000f48, + 0xf4900000f4d, + 0xf4e00000f52, + 0xf5300000f57, + 0xf5800000f5c, + 0xf5d00000f69, + 0xf6a00000f6d, + 0xf7100000f73, + 0xf7400000f75, + 0xf7a00000f81, + 0xf8200000f85, + 0xf8600000f93, + 0xf9400000f98, + 0xf9900000f9d, + 0xf9e00000fa2, + 0xfa300000fa7, + 0xfa800000fac, + 0xfad00000fb9, + 0xfba00000fbd, + 0xfc600000fc7, + 0x10000000104a, + 0x10500000109e, + 0x10d0000010fb, + 0x10fd00001100, + 0x120000001249, + 0x124a0000124e, + 0x125000001257, + 0x125800001259, + 0x125a0000125e, + 0x126000001289, + 0x128a0000128e, + 0x1290000012b1, + 0x12b2000012b6, + 0x12b8000012bf, + 0x12c0000012c1, + 0x12c2000012c6, + 0x12c8000012d7, + 0x12d800001311, + 0x131200001316, + 0x13180000135b, + 0x135d00001360, + 0x138000001390, + 0x13a0000013f6, + 0x14010000166d, + 0x166f00001680, + 0x16810000169b, + 0x16a0000016eb, + 0x16f1000016f9, + 0x17000000170d, + 0x170e00001715, + 0x172000001735, + 0x174000001754, + 0x17600000176d, + 0x176e00001771, + 0x177200001774, + 0x1780000017b4, + 0x17b6000017d4, + 0x17d7000017d8, + 0x17dc000017de, + 0x17e0000017ea, + 0x18100000181a, + 0x182000001879, + 0x1880000018ab, + 0x18b0000018f6, + 0x19000000191f, + 0x19200000192c, + 0x19300000193c, + 0x19460000196e, + 0x197000001975, + 0x1980000019ac, + 0x19b0000019ca, + 0x19d0000019da, + 0x1a0000001a1c, + 0x1a2000001a5f, + 0x1a6000001a7d, + 0x1a7f00001a8a, + 0x1a9000001a9a, + 0x1aa700001aa8, + 0x1ab000001abe, + 0x1abf00001ac1, + 0x1b0000001b4c, + 0x1b5000001b5a, + 0x1b6b00001b74, + 0x1b8000001bf4, + 0x1c0000001c38, + 0x1c4000001c4a, + 0x1c4d00001c7e, + 0x1cd000001cd3, + 0x1cd400001cfb, + 0x1d0000001d2c, + 0x1d2f00001d30, + 0x1d3b00001d3c, + 0x1d4e00001d4f, + 0x1d6b00001d78, + 0x1d7900001d9b, + 0x1dc000001dfa, + 0x1dfb00001e00, + 0x1e0100001e02, + 0x1e0300001e04, + 0x1e0500001e06, + 0x1e0700001e08, + 0x1e0900001e0a, + 0x1e0b00001e0c, + 0x1e0d00001e0e, + 0x1e0f00001e10, + 0x1e1100001e12, + 0x1e1300001e14, + 0x1e1500001e16, + 0x1e1700001e18, + 0x1e1900001e1a, + 0x1e1b00001e1c, + 0x1e1d00001e1e, + 0x1e1f00001e20, + 0x1e2100001e22, + 0x1e2300001e24, + 0x1e2500001e26, + 0x1e2700001e28, + 0x1e2900001e2a, + 0x1e2b00001e2c, + 0x1e2d00001e2e, + 0x1e2f00001e30, + 0x1e3100001e32, + 0x1e3300001e34, + 0x1e3500001e36, + 0x1e3700001e38, + 0x1e3900001e3a, + 0x1e3b00001e3c, + 0x1e3d00001e3e, + 0x1e3f00001e40, + 0x1e4100001e42, + 0x1e4300001e44, + 0x1e4500001e46, + 0x1e4700001e48, + 0x1e4900001e4a, + 0x1e4b00001e4c, + 0x1e4d00001e4e, + 0x1e4f00001e50, + 0x1e5100001e52, + 0x1e5300001e54, + 0x1e5500001e56, + 0x1e5700001e58, + 0x1e5900001e5a, + 0x1e5b00001e5c, + 0x1e5d00001e5e, + 0x1e5f00001e60, + 0x1e6100001e62, + 0x1e6300001e64, + 0x1e6500001e66, + 0x1e6700001e68, + 0x1e6900001e6a, + 0x1e6b00001e6c, + 0x1e6d00001e6e, + 0x1e6f00001e70, + 0x1e7100001e72, + 0x1e7300001e74, + 0x1e7500001e76, + 0x1e7700001e78, + 0x1e7900001e7a, + 0x1e7b00001e7c, + 0x1e7d00001e7e, + 0x1e7f00001e80, + 0x1e8100001e82, + 0x1e8300001e84, + 0x1e8500001e86, + 0x1e8700001e88, + 0x1e8900001e8a, + 0x1e8b00001e8c, + 0x1e8d00001e8e, + 0x1e8f00001e90, + 0x1e9100001e92, + 0x1e9300001e94, + 0x1e9500001e9a, + 0x1e9c00001e9e, + 0x1e9f00001ea0, + 0x1ea100001ea2, + 0x1ea300001ea4, + 0x1ea500001ea6, + 0x1ea700001ea8, + 0x1ea900001eaa, + 0x1eab00001eac, + 0x1ead00001eae, + 0x1eaf00001eb0, + 0x1eb100001eb2, + 0x1eb300001eb4, + 0x1eb500001eb6, + 0x1eb700001eb8, + 0x1eb900001eba, + 0x1ebb00001ebc, + 0x1ebd00001ebe, + 0x1ebf00001ec0, + 0x1ec100001ec2, + 0x1ec300001ec4, + 0x1ec500001ec6, + 0x1ec700001ec8, + 0x1ec900001eca, + 0x1ecb00001ecc, + 0x1ecd00001ece, + 0x1ecf00001ed0, + 0x1ed100001ed2, + 0x1ed300001ed4, + 0x1ed500001ed6, + 0x1ed700001ed8, + 0x1ed900001eda, + 0x1edb00001edc, + 0x1edd00001ede, + 0x1edf00001ee0, + 0x1ee100001ee2, + 0x1ee300001ee4, + 0x1ee500001ee6, + 0x1ee700001ee8, + 0x1ee900001eea, + 0x1eeb00001eec, + 0x1eed00001eee, + 0x1eef00001ef0, + 0x1ef100001ef2, + 0x1ef300001ef4, + 0x1ef500001ef6, + 0x1ef700001ef8, + 0x1ef900001efa, + 0x1efb00001efc, + 0x1efd00001efe, + 0x1eff00001f08, + 0x1f1000001f16, + 0x1f2000001f28, + 0x1f3000001f38, + 0x1f4000001f46, + 0x1f5000001f58, + 0x1f6000001f68, + 0x1f7000001f71, + 0x1f7200001f73, + 0x1f7400001f75, + 0x1f7600001f77, + 0x1f7800001f79, + 0x1f7a00001f7b, + 0x1f7c00001f7d, + 0x1fb000001fb2, + 0x1fb600001fb7, + 0x1fc600001fc7, + 0x1fd000001fd3, + 0x1fd600001fd8, + 0x1fe000001fe3, + 0x1fe400001fe8, + 0x1ff600001ff7, + 0x214e0000214f, + 0x218400002185, + 0x2c3000002c5f, + 0x2c6100002c62, + 0x2c6500002c67, + 0x2c6800002c69, + 0x2c6a00002c6b, + 0x2c6c00002c6d, + 0x2c7100002c72, + 0x2c7300002c75, + 0x2c7600002c7c, + 0x2c8100002c82, + 0x2c8300002c84, + 0x2c8500002c86, + 0x2c8700002c88, + 0x2c8900002c8a, + 0x2c8b00002c8c, + 0x2c8d00002c8e, + 0x2c8f00002c90, + 0x2c9100002c92, + 0x2c9300002c94, + 0x2c9500002c96, + 0x2c9700002c98, + 0x2c9900002c9a, + 0x2c9b00002c9c, + 0x2c9d00002c9e, + 0x2c9f00002ca0, + 0x2ca100002ca2, + 0x2ca300002ca4, + 0x2ca500002ca6, + 0x2ca700002ca8, + 0x2ca900002caa, + 0x2cab00002cac, + 0x2cad00002cae, + 0x2caf00002cb0, + 0x2cb100002cb2, + 0x2cb300002cb4, + 0x2cb500002cb6, + 0x2cb700002cb8, + 0x2cb900002cba, + 0x2cbb00002cbc, + 0x2cbd00002cbe, + 0x2cbf00002cc0, + 0x2cc100002cc2, + 0x2cc300002cc4, + 0x2cc500002cc6, + 0x2cc700002cc8, + 0x2cc900002cca, + 0x2ccb00002ccc, + 0x2ccd00002cce, + 0x2ccf00002cd0, + 0x2cd100002cd2, + 0x2cd300002cd4, + 0x2cd500002cd6, + 0x2cd700002cd8, + 0x2cd900002cda, + 0x2cdb00002cdc, + 0x2cdd00002cde, + 0x2cdf00002ce0, + 0x2ce100002ce2, + 0x2ce300002ce5, + 0x2cec00002ced, + 0x2cee00002cf2, + 0x2cf300002cf4, + 0x2d0000002d26, + 0x2d2700002d28, + 0x2d2d00002d2e, + 0x2d3000002d68, + 0x2d7f00002d97, + 0x2da000002da7, + 0x2da800002daf, + 0x2db000002db7, + 0x2db800002dbf, + 0x2dc000002dc7, + 0x2dc800002dcf, + 0x2dd000002dd7, + 0x2dd800002ddf, + 0x2de000002e00, + 0x2e2f00002e30, + 0x300500003008, + 0x302a0000302e, + 0x303c0000303d, + 0x304100003097, + 0x30990000309b, + 0x309d0000309f, + 0x30a1000030fb, + 0x30fc000030ff, + 0x310500003130, + 0x31a0000031c0, + 0x31f000003200, + 0x340000004dc0, + 0x4e0000009ffd, + 0xa0000000a48d, + 0xa4d00000a4fe, + 0xa5000000a60d, + 0xa6100000a62c, + 0xa6410000a642, + 0xa6430000a644, + 0xa6450000a646, + 0xa6470000a648, + 0xa6490000a64a, + 0xa64b0000a64c, + 0xa64d0000a64e, + 0xa64f0000a650, + 0xa6510000a652, + 0xa6530000a654, + 0xa6550000a656, + 0xa6570000a658, + 0xa6590000a65a, + 0xa65b0000a65c, + 0xa65d0000a65e, + 0xa65f0000a660, + 0xa6610000a662, + 0xa6630000a664, + 0xa6650000a666, + 0xa6670000a668, + 0xa6690000a66a, + 0xa66b0000a66c, + 0xa66d0000a670, + 0xa6740000a67e, + 0xa67f0000a680, + 0xa6810000a682, + 0xa6830000a684, + 0xa6850000a686, + 0xa6870000a688, + 0xa6890000a68a, + 0xa68b0000a68c, + 0xa68d0000a68e, + 0xa68f0000a690, + 0xa6910000a692, + 0xa6930000a694, + 0xa6950000a696, + 0xa6970000a698, + 0xa6990000a69a, + 0xa69b0000a69c, + 0xa69e0000a6e6, + 0xa6f00000a6f2, + 0xa7170000a720, + 0xa7230000a724, + 0xa7250000a726, + 0xa7270000a728, + 0xa7290000a72a, + 0xa72b0000a72c, + 0xa72d0000a72e, + 0xa72f0000a732, + 0xa7330000a734, + 0xa7350000a736, + 0xa7370000a738, + 0xa7390000a73a, + 0xa73b0000a73c, + 0xa73d0000a73e, + 0xa73f0000a740, + 0xa7410000a742, + 0xa7430000a744, + 0xa7450000a746, + 0xa7470000a748, + 0xa7490000a74a, + 0xa74b0000a74c, + 0xa74d0000a74e, + 0xa74f0000a750, + 0xa7510000a752, + 0xa7530000a754, + 0xa7550000a756, + 0xa7570000a758, + 0xa7590000a75a, + 0xa75b0000a75c, + 0xa75d0000a75e, + 0xa75f0000a760, + 0xa7610000a762, + 0xa7630000a764, + 0xa7650000a766, + 0xa7670000a768, + 0xa7690000a76a, + 0xa76b0000a76c, + 0xa76d0000a76e, + 0xa76f0000a770, + 0xa7710000a779, + 0xa77a0000a77b, + 0xa77c0000a77d, + 0xa77f0000a780, + 0xa7810000a782, + 0xa7830000a784, + 0xa7850000a786, + 0xa7870000a789, + 0xa78c0000a78d, + 0xa78e0000a790, + 0xa7910000a792, + 0xa7930000a796, + 0xa7970000a798, + 0xa7990000a79a, + 0xa79b0000a79c, + 0xa79d0000a79e, + 0xa79f0000a7a0, + 0xa7a10000a7a2, + 0xa7a30000a7a4, + 0xa7a50000a7a6, + 0xa7a70000a7a8, + 0xa7a90000a7aa, + 0xa7af0000a7b0, + 0xa7b50000a7b6, + 0xa7b70000a7b8, + 0xa7b90000a7ba, + 0xa7bb0000a7bc, + 0xa7bd0000a7be, + 0xa7bf0000a7c0, + 0xa7c30000a7c4, + 0xa7c80000a7c9, + 0xa7ca0000a7cb, + 0xa7f60000a7f8, + 0xa7fa0000a828, + 0xa82c0000a82d, + 0xa8400000a874, + 0xa8800000a8c6, + 0xa8d00000a8da, + 0xa8e00000a8f8, + 0xa8fb0000a8fc, + 0xa8fd0000a92e, + 0xa9300000a954, + 0xa9800000a9c1, + 0xa9cf0000a9da, + 0xa9e00000a9ff, + 0xaa000000aa37, + 0xaa400000aa4e, + 0xaa500000aa5a, + 0xaa600000aa77, + 0xaa7a0000aac3, + 0xaadb0000aade, + 0xaae00000aaf0, + 0xaaf20000aaf7, + 0xab010000ab07, + 0xab090000ab0f, + 0xab110000ab17, + 0xab200000ab27, + 0xab280000ab2f, + 0xab300000ab5b, + 0xab600000ab6a, + 0xabc00000abeb, + 0xabec0000abee, + 0xabf00000abfa, + 0xac000000d7a4, + 0xfa0e0000fa10, + 0xfa110000fa12, + 0xfa130000fa15, + 0xfa1f0000fa20, + 0xfa210000fa22, + 0xfa230000fa25, + 0xfa270000fa2a, + 0xfb1e0000fb1f, + 0xfe200000fe30, + 0xfe730000fe74, + 0x100000001000c, + 0x1000d00010027, + 0x100280001003b, + 0x1003c0001003e, + 0x1003f0001004e, + 0x100500001005e, + 0x10080000100fb, + 0x101fd000101fe, + 0x102800001029d, + 0x102a0000102d1, + 0x102e0000102e1, + 0x1030000010320, + 0x1032d00010341, + 0x103420001034a, + 0x103500001037b, + 0x103800001039e, + 0x103a0000103c4, + 0x103c8000103d0, + 0x104280001049e, + 0x104a0000104aa, + 0x104d8000104fc, + 0x1050000010528, + 0x1053000010564, + 0x1060000010737, + 0x1074000010756, + 0x1076000010768, + 0x1080000010806, + 0x1080800010809, + 0x1080a00010836, + 0x1083700010839, + 0x1083c0001083d, + 0x1083f00010856, + 0x1086000010877, + 0x108800001089f, + 0x108e0000108f3, + 0x108f4000108f6, + 0x1090000010916, + 0x109200001093a, + 0x10980000109b8, + 0x109be000109c0, + 0x10a0000010a04, + 0x10a0500010a07, + 0x10a0c00010a14, + 0x10a1500010a18, + 0x10a1900010a36, + 0x10a3800010a3b, + 0x10a3f00010a40, + 0x10a6000010a7d, + 0x10a8000010a9d, + 0x10ac000010ac8, + 0x10ac900010ae7, + 0x10b0000010b36, + 0x10b4000010b56, + 0x10b6000010b73, + 0x10b8000010b92, + 0x10c0000010c49, + 0x10cc000010cf3, + 0x10d0000010d28, + 0x10d3000010d3a, + 0x10e8000010eaa, + 0x10eab00010ead, + 0x10eb000010eb2, + 0x10f0000010f1d, + 0x10f2700010f28, + 0x10f3000010f51, + 0x10fb000010fc5, + 0x10fe000010ff7, + 0x1100000011047, + 0x1106600011070, + 0x1107f000110bb, + 0x110d0000110e9, + 0x110f0000110fa, + 0x1110000011135, + 0x1113600011140, + 0x1114400011148, + 0x1115000011174, + 0x1117600011177, + 0x11180000111c5, + 0x111c9000111cd, + 0x111ce000111db, + 0x111dc000111dd, + 0x1120000011212, + 0x1121300011238, + 0x1123e0001123f, + 0x1128000011287, + 0x1128800011289, + 0x1128a0001128e, + 0x1128f0001129e, + 0x1129f000112a9, + 0x112b0000112eb, + 0x112f0000112fa, + 0x1130000011304, + 0x113050001130d, + 0x1130f00011311, + 0x1131300011329, + 0x1132a00011331, + 0x1133200011334, + 0x113350001133a, + 0x1133b00011345, + 0x1134700011349, + 0x1134b0001134e, + 0x1135000011351, + 0x1135700011358, + 0x1135d00011364, + 0x113660001136d, + 0x1137000011375, + 0x114000001144b, + 0x114500001145a, + 0x1145e00011462, + 0x11480000114c6, + 0x114c7000114c8, + 0x114d0000114da, + 0x11580000115b6, + 0x115b8000115c1, + 0x115d8000115de, + 0x1160000011641, + 0x1164400011645, + 0x116500001165a, + 0x11680000116b9, + 0x116c0000116ca, + 0x117000001171b, + 0x1171d0001172c, + 0x117300001173a, + 0x118000001183b, + 0x118c0000118ea, + 0x118ff00011907, + 0x119090001190a, + 0x1190c00011914, + 0x1191500011917, + 0x1191800011936, + 0x1193700011939, + 0x1193b00011944, + 0x119500001195a, + 0x119a0000119a8, + 0x119aa000119d8, + 0x119da000119e2, + 0x119e3000119e5, + 0x11a0000011a3f, + 0x11a4700011a48, + 0x11a5000011a9a, + 0x11a9d00011a9e, + 0x11ac000011af9, + 0x11c0000011c09, + 0x11c0a00011c37, + 0x11c3800011c41, + 0x11c5000011c5a, + 0x11c7200011c90, + 0x11c9200011ca8, + 0x11ca900011cb7, + 0x11d0000011d07, + 0x11d0800011d0a, + 0x11d0b00011d37, + 0x11d3a00011d3b, + 0x11d3c00011d3e, + 0x11d3f00011d48, + 0x11d5000011d5a, + 0x11d6000011d66, + 0x11d6700011d69, + 0x11d6a00011d8f, + 0x11d9000011d92, + 0x11d9300011d99, + 0x11da000011daa, + 0x11ee000011ef7, + 0x11fb000011fb1, + 0x120000001239a, + 0x1248000012544, + 0x130000001342f, + 0x1440000014647, + 0x1680000016a39, + 0x16a4000016a5f, + 0x16a6000016a6a, + 0x16ad000016aee, + 0x16af000016af5, + 0x16b0000016b37, + 0x16b4000016b44, + 0x16b5000016b5a, + 0x16b6300016b78, + 0x16b7d00016b90, + 0x16e6000016e80, + 0x16f0000016f4b, + 0x16f4f00016f88, + 0x16f8f00016fa0, + 0x16fe000016fe2, + 0x16fe300016fe5, + 0x16ff000016ff2, + 0x17000000187f8, + 0x1880000018cd6, + 0x18d0000018d09, + 0x1b0000001b11f, + 0x1b1500001b153, + 0x1b1640001b168, + 0x1b1700001b2fc, + 0x1bc000001bc6b, + 0x1bc700001bc7d, + 0x1bc800001bc89, + 0x1bc900001bc9a, + 0x1bc9d0001bc9f, + 0x1da000001da37, + 0x1da3b0001da6d, + 0x1da750001da76, + 0x1da840001da85, + 0x1da9b0001daa0, + 0x1daa10001dab0, + 0x1e0000001e007, + 0x1e0080001e019, + 0x1e01b0001e022, + 0x1e0230001e025, + 0x1e0260001e02b, + 0x1e1000001e12d, + 0x1e1300001e13e, + 0x1e1400001e14a, + 0x1e14e0001e14f, + 0x1e2c00001e2fa, + 0x1e8000001e8c5, + 0x1e8d00001e8d7, + 0x1e9220001e94c, + 0x1e9500001e95a, + 0x1fbf00001fbfa, + 0x200000002a6de, + 0x2a7000002b735, + 0x2b7400002b81e, + 0x2b8200002cea2, + 0x2ceb00002ebe1, + 0x300000003134b, + ), + 'CONTEXTJ': ( + 0x200c0000200e, + ), + 'CONTEXTO': ( + 0xb7000000b8, + 0x37500000376, + 0x5f3000005f5, + 0x6600000066a, + 0x6f0000006fa, + 0x30fb000030fc, + ), +} diff --git a/WebCrawler/venv/Lib/site-packages/idna/intranges.py b/WebCrawler/venv/Lib/site-packages/idna/intranges.py new file mode 100644 index 0000000..fa8a735 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/idna/intranges.py @@ -0,0 +1,53 @@ +""" +Given a list of integers, made up of (hopefully) a small number of long runs +of consecutive integers, compute a representation of the form +((start1, end1), (start2, end2) ...). Then answer the question "was x present +in the original list?" in time O(log(# runs)). +""" + +import bisect + +def intranges_from_list(list_): + """Represent a list of integers as a sequence of ranges: + ((start_0, end_0), (start_1, end_1), ...), such that the original + integers are exactly those x such that start_i <= x < end_i for some i. + + Ranges are encoded as single integers (start << 32 | end), not as tuples. + """ + + sorted_list = sorted(list_) + ranges = [] + last_write = -1 + for i in range(len(sorted_list)): + if i+1 < len(sorted_list): + if sorted_list[i] == sorted_list[i+1]-1: + continue + current_range = sorted_list[last_write+1:i+1] + ranges.append(_encode_range(current_range[0], current_range[-1] + 1)) + last_write = i + + return tuple(ranges) + +def _encode_range(start, end): + return (start << 32) | end + +def _decode_range(r): + return (r >> 32), (r & ((1 << 32) - 1)) + + +def intranges_contain(int_, ranges): + """Determine if `int_` falls into one of the ranges in `ranges`.""" + tuple_ = _encode_range(int_, 0) + pos = bisect.bisect_left(ranges, tuple_) + # we could be immediately ahead of a tuple (start, end) + # with start < int_ <= end + if pos > 0: + left, right = _decode_range(ranges[pos-1]) + if left <= int_ < right: + return True + # or we could be immediately behind a tuple (int_, end) + if pos < len(ranges): + left, _ = _decode_range(ranges[pos]) + if left == int_: + return True + return False diff --git a/WebCrawler/venv/Lib/site-packages/idna/package_data.py b/WebCrawler/venv/Lib/site-packages/idna/package_data.py new file mode 100644 index 0000000..ce1c521 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/idna/package_data.py @@ -0,0 +1,2 @@ +__version__ = '2.10' + diff --git a/WebCrawler/venv/Lib/site-packages/idna/uts46data.py b/WebCrawler/venv/Lib/site-packages/idna/uts46data.py new file mode 100644 index 0000000..3766dd4 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/idna/uts46data.py @@ -0,0 +1,8357 @@ +# This file is automatically generated by tools/idna-data +# vim: set fileencoding=utf-8 : + +"""IDNA Mapping Table from UTS46.""" + + +__version__ = "13.0.0" +def _seg_0(): + return [ + (0x0, '3'), + (0x1, '3'), + (0x2, '3'), + (0x3, '3'), + (0x4, '3'), + (0x5, '3'), + (0x6, '3'), + (0x7, '3'), + (0x8, '3'), + (0x9, '3'), + (0xA, '3'), + (0xB, '3'), + (0xC, '3'), + (0xD, '3'), + (0xE, '3'), + (0xF, '3'), + (0x10, '3'), + (0x11, '3'), + (0x12, '3'), + (0x13, '3'), + (0x14, '3'), + (0x15, '3'), + (0x16, '3'), + (0x17, '3'), + (0x18, '3'), + (0x19, '3'), + (0x1A, '3'), + (0x1B, '3'), + (0x1C, '3'), + (0x1D, '3'), + (0x1E, '3'), + (0x1F, '3'), + (0x20, '3'), + (0x21, '3'), + (0x22, '3'), + (0x23, '3'), + (0x24, '3'), + (0x25, '3'), + (0x26, '3'), + (0x27, '3'), + (0x28, '3'), + (0x29, '3'), + (0x2A, '3'), + (0x2B, '3'), + (0x2C, '3'), + (0x2D, 'V'), + (0x2E, 'V'), + (0x2F, '3'), + (0x30, 'V'), + (0x31, 'V'), + (0x32, 'V'), + (0x33, 'V'), + (0x34, 'V'), + (0x35, 'V'), + (0x36, 'V'), + (0x37, 'V'), + (0x38, 'V'), + (0x39, 'V'), + (0x3A, '3'), + (0x3B, '3'), + (0x3C, '3'), + (0x3D, '3'), + (0x3E, '3'), + (0x3F, '3'), + (0x40, '3'), + (0x41, 'M', u'a'), + (0x42, 'M', u'b'), + (0x43, 'M', u'c'), + (0x44, 'M', u'd'), + (0x45, 'M', u'e'), + (0x46, 'M', u'f'), + (0x47, 'M', u'g'), + (0x48, 'M', u'h'), + (0x49, 'M', u'i'), + (0x4A, 'M', u'j'), + (0x4B, 'M', u'k'), + (0x4C, 'M', u'l'), + (0x4D, 'M', u'm'), + (0x4E, 'M', u'n'), + (0x4F, 'M', u'o'), + (0x50, 'M', u'p'), + (0x51, 'M', u'q'), + (0x52, 'M', u'r'), + (0x53, 'M', u's'), + (0x54, 'M', u't'), + (0x55, 'M', u'u'), + (0x56, 'M', u'v'), + (0x57, 'M', u'w'), + (0x58, 'M', u'x'), + (0x59, 'M', u'y'), + (0x5A, 'M', u'z'), + (0x5B, '3'), + (0x5C, '3'), + (0x5D, '3'), + (0x5E, '3'), + (0x5F, '3'), + (0x60, '3'), + (0x61, 'V'), + (0x62, 'V'), + (0x63, 'V'), + ] + +def _seg_1(): + return [ + (0x64, 'V'), + (0x65, 'V'), + (0x66, 'V'), + (0x67, 'V'), + (0x68, 'V'), + (0x69, 'V'), + (0x6A, 'V'), + (0x6B, 'V'), + (0x6C, 'V'), + (0x6D, 'V'), + (0x6E, 'V'), + (0x6F, 'V'), + (0x70, 'V'), + (0x71, 'V'), + (0x72, 'V'), + (0x73, 'V'), + (0x74, 'V'), + (0x75, 'V'), + (0x76, 'V'), + (0x77, 'V'), + (0x78, 'V'), + (0x79, 'V'), + (0x7A, 'V'), + (0x7B, '3'), + (0x7C, '3'), + (0x7D, '3'), + (0x7E, '3'), + (0x7F, '3'), + (0x80, 'X'), + (0x81, 'X'), + (0x82, 'X'), + (0x83, 'X'), + (0x84, 'X'), + (0x85, 'X'), + (0x86, 'X'), + (0x87, 'X'), + (0x88, 'X'), + (0x89, 'X'), + (0x8A, 'X'), + (0x8B, 'X'), + (0x8C, 'X'), + (0x8D, 'X'), + (0x8E, 'X'), + (0x8F, 'X'), + (0x90, 'X'), + (0x91, 'X'), + (0x92, 'X'), + (0x93, 'X'), + (0x94, 'X'), + (0x95, 'X'), + (0x96, 'X'), + (0x97, 'X'), + (0x98, 'X'), + (0x99, 'X'), + (0x9A, 'X'), + (0x9B, 'X'), + (0x9C, 'X'), + (0x9D, 'X'), + (0x9E, 'X'), + (0x9F, 'X'), + (0xA0, '3', u' '), + (0xA1, 'V'), + (0xA2, 'V'), + (0xA3, 'V'), + (0xA4, 'V'), + (0xA5, 'V'), + (0xA6, 'V'), + (0xA7, 'V'), + (0xA8, '3', u' ̈'), + (0xA9, 'V'), + (0xAA, 'M', u'a'), + (0xAB, 'V'), + (0xAC, 'V'), + (0xAD, 'I'), + (0xAE, 'V'), + (0xAF, '3', u' ̄'), + (0xB0, 'V'), + (0xB1, 'V'), + (0xB2, 'M', u'2'), + (0xB3, 'M', u'3'), + (0xB4, '3', u' ́'), + (0xB5, 'M', u'μ'), + (0xB6, 'V'), + (0xB7, 'V'), + (0xB8, '3', u' ̧'), + (0xB9, 'M', u'1'), + (0xBA, 'M', u'o'), + (0xBB, 'V'), + (0xBC, 'M', u'1⁄4'), + (0xBD, 'M', u'1⁄2'), + (0xBE, 'M', u'3⁄4'), + (0xBF, 'V'), + (0xC0, 'M', u'à'), + (0xC1, 'M', u'á'), + (0xC2, 'M', u'â'), + (0xC3, 'M', u'ã'), + (0xC4, 'M', u'ä'), + (0xC5, 'M', u'å'), + (0xC6, 'M', u'æ'), + (0xC7, 'M', u'ç'), + ] + +def _seg_2(): + return [ + (0xC8, 'M', u'è'), + (0xC9, 'M', u'é'), + (0xCA, 'M', u'ê'), + (0xCB, 'M', u'ë'), + (0xCC, 'M', u'ì'), + (0xCD, 'M', u'í'), + (0xCE, 'M', u'î'), + (0xCF, 'M', u'ï'), + (0xD0, 'M', u'ð'), + (0xD1, 'M', u'ñ'), + (0xD2, 'M', u'ò'), + (0xD3, 'M', u'ó'), + (0xD4, 'M', u'ô'), + (0xD5, 'M', u'õ'), + (0xD6, 'M', u'ö'), + (0xD7, 'V'), + (0xD8, 'M', u'ø'), + (0xD9, 'M', u'ù'), + (0xDA, 'M', u'ú'), + (0xDB, 'M', u'û'), + (0xDC, 'M', u'ü'), + (0xDD, 'M', u'ý'), + (0xDE, 'M', u'þ'), + (0xDF, 'D', u'ss'), + (0xE0, 'V'), + (0xE1, 'V'), + (0xE2, 'V'), + (0xE3, 'V'), + (0xE4, 'V'), + (0xE5, 'V'), + (0xE6, 'V'), + (0xE7, 'V'), + (0xE8, 'V'), + (0xE9, 'V'), + (0xEA, 'V'), + (0xEB, 'V'), + (0xEC, 'V'), + (0xED, 'V'), + (0xEE, 'V'), + (0xEF, 'V'), + (0xF0, 'V'), + (0xF1, 'V'), + (0xF2, 'V'), + (0xF3, 'V'), + (0xF4, 'V'), + (0xF5, 'V'), + (0xF6, 'V'), + (0xF7, 'V'), + (0xF8, 'V'), + (0xF9, 'V'), + (0xFA, 'V'), + (0xFB, 'V'), + (0xFC, 'V'), + (0xFD, 'V'), + (0xFE, 'V'), + (0xFF, 'V'), + (0x100, 'M', u'ā'), + (0x101, 'V'), + (0x102, 'M', u'ă'), + (0x103, 'V'), + (0x104, 'M', u'ą'), + (0x105, 'V'), + (0x106, 'M', u'ć'), + (0x107, 'V'), + (0x108, 'M', u'ĉ'), + (0x109, 'V'), + (0x10A, 'M', u'ċ'), + (0x10B, 'V'), + (0x10C, 'M', u'č'), + (0x10D, 'V'), + (0x10E, 'M', u'ď'), + (0x10F, 'V'), + (0x110, 'M', u'đ'), + (0x111, 'V'), + (0x112, 'M', u'ē'), + (0x113, 'V'), + (0x114, 'M', u'ĕ'), + (0x115, 'V'), + (0x116, 'M', u'ė'), + (0x117, 'V'), + (0x118, 'M', u'ę'), + (0x119, 'V'), + (0x11A, 'M', u'ě'), + (0x11B, 'V'), + (0x11C, 'M', u'ĝ'), + (0x11D, 'V'), + (0x11E, 'M', u'ğ'), + (0x11F, 'V'), + (0x120, 'M', u'ġ'), + (0x121, 'V'), + (0x122, 'M', u'ģ'), + (0x123, 'V'), + (0x124, 'M', u'ĥ'), + (0x125, 'V'), + (0x126, 'M', u'ħ'), + (0x127, 'V'), + (0x128, 'M', u'ĩ'), + (0x129, 'V'), + (0x12A, 'M', u'ī'), + (0x12B, 'V'), + ] + +def _seg_3(): + return [ + (0x12C, 'M', u'ĭ'), + (0x12D, 'V'), + (0x12E, 'M', u'į'), + (0x12F, 'V'), + (0x130, 'M', u'i̇'), + (0x131, 'V'), + (0x132, 'M', u'ij'), + (0x134, 'M', u'ĵ'), + (0x135, 'V'), + (0x136, 'M', u'ķ'), + (0x137, 'V'), + (0x139, 'M', u'ĺ'), + (0x13A, 'V'), + (0x13B, 'M', u'ļ'), + (0x13C, 'V'), + (0x13D, 'M', u'ľ'), + (0x13E, 'V'), + (0x13F, 'M', u'l·'), + (0x141, 'M', u'ł'), + (0x142, 'V'), + (0x143, 'M', u'ń'), + (0x144, 'V'), + (0x145, 'M', u'ņ'), + (0x146, 'V'), + (0x147, 'M', u'ň'), + (0x148, 'V'), + (0x149, 'M', u'ʼn'), + (0x14A, 'M', u'ŋ'), + (0x14B, 'V'), + (0x14C, 'M', u'ō'), + (0x14D, 'V'), + (0x14E, 'M', u'ŏ'), + (0x14F, 'V'), + (0x150, 'M', u'ő'), + (0x151, 'V'), + (0x152, 'M', u'œ'), + (0x153, 'V'), + (0x154, 'M', u'ŕ'), + (0x155, 'V'), + (0x156, 'M', u'ŗ'), + (0x157, 'V'), + (0x158, 'M', u'ř'), + (0x159, 'V'), + (0x15A, 'M', u'ś'), + (0x15B, 'V'), + (0x15C, 'M', u'ŝ'), + (0x15D, 'V'), + (0x15E, 'M', u'ş'), + (0x15F, 'V'), + (0x160, 'M', u'š'), + (0x161, 'V'), + (0x162, 'M', u'ţ'), + (0x163, 'V'), + (0x164, 'M', u'ť'), + (0x165, 'V'), + (0x166, 'M', u'ŧ'), + (0x167, 'V'), + (0x168, 'M', u'ũ'), + (0x169, 'V'), + (0x16A, 'M', u'ū'), + (0x16B, 'V'), + (0x16C, 'M', u'ŭ'), + (0x16D, 'V'), + (0x16E, 'M', u'ů'), + (0x16F, 'V'), + (0x170, 'M', u'ű'), + (0x171, 'V'), + (0x172, 'M', u'ų'), + (0x173, 'V'), + (0x174, 'M', u'ŵ'), + (0x175, 'V'), + (0x176, 'M', u'ŷ'), + (0x177, 'V'), + (0x178, 'M', u'ÿ'), + (0x179, 'M', u'ź'), + (0x17A, 'V'), + (0x17B, 'M', u'ż'), + (0x17C, 'V'), + (0x17D, 'M', u'ž'), + (0x17E, 'V'), + (0x17F, 'M', u's'), + (0x180, 'V'), + (0x181, 'M', u'ɓ'), + (0x182, 'M', u'ƃ'), + (0x183, 'V'), + (0x184, 'M', u'ƅ'), + (0x185, 'V'), + (0x186, 'M', u'ɔ'), + (0x187, 'M', u'ƈ'), + (0x188, 'V'), + (0x189, 'M', u'ɖ'), + (0x18A, 'M', u'ɗ'), + (0x18B, 'M', u'ƌ'), + (0x18C, 'V'), + (0x18E, 'M', u'ǝ'), + (0x18F, 'M', u'ə'), + (0x190, 'M', u'ɛ'), + (0x191, 'M', u'ƒ'), + (0x192, 'V'), + (0x193, 'M', u'ɠ'), + ] + +def _seg_4(): + return [ + (0x194, 'M', u'ɣ'), + (0x195, 'V'), + (0x196, 'M', u'ɩ'), + (0x197, 'M', u'ɨ'), + (0x198, 'M', u'ƙ'), + (0x199, 'V'), + (0x19C, 'M', u'ɯ'), + (0x19D, 'M', u'ɲ'), + (0x19E, 'V'), + (0x19F, 'M', u'ɵ'), + (0x1A0, 'M', u'ơ'), + (0x1A1, 'V'), + (0x1A2, 'M', u'ƣ'), + (0x1A3, 'V'), + (0x1A4, 'M', u'ƥ'), + (0x1A5, 'V'), + (0x1A6, 'M', u'ʀ'), + (0x1A7, 'M', u'ƨ'), + (0x1A8, 'V'), + (0x1A9, 'M', u'ʃ'), + (0x1AA, 'V'), + (0x1AC, 'M', u'ƭ'), + (0x1AD, 'V'), + (0x1AE, 'M', u'ʈ'), + (0x1AF, 'M', u'ư'), + (0x1B0, 'V'), + (0x1B1, 'M', u'ʊ'), + (0x1B2, 'M', u'ʋ'), + (0x1B3, 'M', u'ƴ'), + (0x1B4, 'V'), + (0x1B5, 'M', u'ƶ'), + (0x1B6, 'V'), + (0x1B7, 'M', u'ʒ'), + (0x1B8, 'M', u'ƹ'), + (0x1B9, 'V'), + (0x1BC, 'M', u'ƽ'), + (0x1BD, 'V'), + (0x1C4, 'M', u'dž'), + (0x1C7, 'M', u'lj'), + (0x1CA, 'M', u'nj'), + (0x1CD, 'M', u'ǎ'), + (0x1CE, 'V'), + (0x1CF, 'M', u'ǐ'), + (0x1D0, 'V'), + (0x1D1, 'M', u'ǒ'), + (0x1D2, 'V'), + (0x1D3, 'M', u'ǔ'), + (0x1D4, 'V'), + (0x1D5, 'M', u'ǖ'), + (0x1D6, 'V'), + (0x1D7, 'M', u'ǘ'), + (0x1D8, 'V'), + (0x1D9, 'M', u'ǚ'), + (0x1DA, 'V'), + (0x1DB, 'M', u'ǜ'), + (0x1DC, 'V'), + (0x1DE, 'M', u'ǟ'), + (0x1DF, 'V'), + (0x1E0, 'M', u'ǡ'), + (0x1E1, 'V'), + (0x1E2, 'M', u'ǣ'), + (0x1E3, 'V'), + (0x1E4, 'M', u'ǥ'), + (0x1E5, 'V'), + (0x1E6, 'M', u'ǧ'), + (0x1E7, 'V'), + (0x1E8, 'M', u'ǩ'), + (0x1E9, 'V'), + (0x1EA, 'M', u'ǫ'), + (0x1EB, 'V'), + (0x1EC, 'M', u'ǭ'), + (0x1ED, 'V'), + (0x1EE, 'M', u'ǯ'), + (0x1EF, 'V'), + (0x1F1, 'M', u'dz'), + (0x1F4, 'M', u'ǵ'), + (0x1F5, 'V'), + (0x1F6, 'M', u'ƕ'), + (0x1F7, 'M', u'ƿ'), + (0x1F8, 'M', u'ǹ'), + (0x1F9, 'V'), + (0x1FA, 'M', u'ǻ'), + (0x1FB, 'V'), + (0x1FC, 'M', u'ǽ'), + (0x1FD, 'V'), + (0x1FE, 'M', u'ǿ'), + (0x1FF, 'V'), + (0x200, 'M', u'ȁ'), + (0x201, 'V'), + (0x202, 'M', u'ȃ'), + (0x203, 'V'), + (0x204, 'M', u'ȅ'), + (0x205, 'V'), + (0x206, 'M', u'ȇ'), + (0x207, 'V'), + (0x208, 'M', u'ȉ'), + (0x209, 'V'), + (0x20A, 'M', u'ȋ'), + (0x20B, 'V'), + (0x20C, 'M', u'ȍ'), + ] + +def _seg_5(): + return [ + (0x20D, 'V'), + (0x20E, 'M', u'ȏ'), + (0x20F, 'V'), + (0x210, 'M', u'ȑ'), + (0x211, 'V'), + (0x212, 'M', u'ȓ'), + (0x213, 'V'), + (0x214, 'M', u'ȕ'), + (0x215, 'V'), + (0x216, 'M', u'ȗ'), + (0x217, 'V'), + (0x218, 'M', u'ș'), + (0x219, 'V'), + (0x21A, 'M', u'ț'), + (0x21B, 'V'), + (0x21C, 'M', u'ȝ'), + (0x21D, 'V'), + (0x21E, 'M', u'ȟ'), + (0x21F, 'V'), + (0x220, 'M', u'ƞ'), + (0x221, 'V'), + (0x222, 'M', u'ȣ'), + (0x223, 'V'), + (0x224, 'M', u'ȥ'), + (0x225, 'V'), + (0x226, 'M', u'ȧ'), + (0x227, 'V'), + (0x228, 'M', u'ȩ'), + (0x229, 'V'), + (0x22A, 'M', u'ȫ'), + (0x22B, 'V'), + (0x22C, 'M', u'ȭ'), + (0x22D, 'V'), + (0x22E, 'M', u'ȯ'), + (0x22F, 'V'), + (0x230, 'M', u'ȱ'), + (0x231, 'V'), + (0x232, 'M', u'ȳ'), + (0x233, 'V'), + (0x23A, 'M', u'ⱥ'), + (0x23B, 'M', u'ȼ'), + (0x23C, 'V'), + (0x23D, 'M', u'ƚ'), + (0x23E, 'M', u'ⱦ'), + (0x23F, 'V'), + (0x241, 'M', u'ɂ'), + (0x242, 'V'), + (0x243, 'M', u'ƀ'), + (0x244, 'M', u'ʉ'), + (0x245, 'M', u'ʌ'), + (0x246, 'M', u'ɇ'), + (0x247, 'V'), + (0x248, 'M', u'ɉ'), + (0x249, 'V'), + (0x24A, 'M', u'ɋ'), + (0x24B, 'V'), + (0x24C, 'M', u'ɍ'), + (0x24D, 'V'), + (0x24E, 'M', u'ɏ'), + (0x24F, 'V'), + (0x2B0, 'M', u'h'), + (0x2B1, 'M', u'ɦ'), + (0x2B2, 'M', u'j'), + (0x2B3, 'M', u'r'), + (0x2B4, 'M', u'ɹ'), + (0x2B5, 'M', u'ɻ'), + (0x2B6, 'M', u'ʁ'), + (0x2B7, 'M', u'w'), + (0x2B8, 'M', u'y'), + (0x2B9, 'V'), + (0x2D8, '3', u' ̆'), + (0x2D9, '3', u' ̇'), + (0x2DA, '3', u' ̊'), + (0x2DB, '3', u' ̨'), + (0x2DC, '3', u' ̃'), + (0x2DD, '3', u' ̋'), + (0x2DE, 'V'), + (0x2E0, 'M', u'ɣ'), + (0x2E1, 'M', u'l'), + (0x2E2, 'M', u's'), + (0x2E3, 'M', u'x'), + (0x2E4, 'M', u'ʕ'), + (0x2E5, 'V'), + (0x340, 'M', u'̀'), + (0x341, 'M', u'́'), + (0x342, 'V'), + (0x343, 'M', u'̓'), + (0x344, 'M', u'̈́'), + (0x345, 'M', u'ι'), + (0x346, 'V'), + (0x34F, 'I'), + (0x350, 'V'), + (0x370, 'M', u'ͱ'), + (0x371, 'V'), + (0x372, 'M', u'ͳ'), + (0x373, 'V'), + (0x374, 'M', u'ʹ'), + (0x375, 'V'), + (0x376, 'M', u'ͷ'), + (0x377, 'V'), + ] + +def _seg_6(): + return [ + (0x378, 'X'), + (0x37A, '3', u' ι'), + (0x37B, 'V'), + (0x37E, '3', u';'), + (0x37F, 'M', u'ϳ'), + (0x380, 'X'), + (0x384, '3', u' ́'), + (0x385, '3', u' ̈́'), + (0x386, 'M', u'ά'), + (0x387, 'M', u'·'), + (0x388, 'M', u'έ'), + (0x389, 'M', u'ή'), + (0x38A, 'M', u'ί'), + (0x38B, 'X'), + (0x38C, 'M', u'ό'), + (0x38D, 'X'), + (0x38E, 'M', u'ύ'), + (0x38F, 'M', u'ώ'), + (0x390, 'V'), + (0x391, 'M', u'α'), + (0x392, 'M', u'β'), + (0x393, 'M', u'γ'), + (0x394, 'M', u'δ'), + (0x395, 'M', u'ε'), + (0x396, 'M', u'ζ'), + (0x397, 'M', u'η'), + (0x398, 'M', u'θ'), + (0x399, 'M', u'ι'), + (0x39A, 'M', u'κ'), + (0x39B, 'M', u'λ'), + (0x39C, 'M', u'μ'), + (0x39D, 'M', u'ν'), + (0x39E, 'M', u'ξ'), + (0x39F, 'M', u'ο'), + (0x3A0, 'M', u'π'), + (0x3A1, 'M', u'ρ'), + (0x3A2, 'X'), + (0x3A3, 'M', u'σ'), + (0x3A4, 'M', u'τ'), + (0x3A5, 'M', u'υ'), + (0x3A6, 'M', u'φ'), + (0x3A7, 'M', u'χ'), + (0x3A8, 'M', u'ψ'), + (0x3A9, 'M', u'ω'), + (0x3AA, 'M', u'ϊ'), + (0x3AB, 'M', u'ϋ'), + (0x3AC, 'V'), + (0x3C2, 'D', u'σ'), + (0x3C3, 'V'), + (0x3CF, 'M', u'ϗ'), + (0x3D0, 'M', u'β'), + (0x3D1, 'M', u'θ'), + (0x3D2, 'M', u'υ'), + (0x3D3, 'M', u'ύ'), + (0x3D4, 'M', u'ϋ'), + (0x3D5, 'M', u'φ'), + (0x3D6, 'M', u'π'), + (0x3D7, 'V'), + (0x3D8, 'M', u'ϙ'), + (0x3D9, 'V'), + (0x3DA, 'M', u'ϛ'), + (0x3DB, 'V'), + (0x3DC, 'M', u'ϝ'), + (0x3DD, 'V'), + (0x3DE, 'M', u'ϟ'), + (0x3DF, 'V'), + (0x3E0, 'M', u'ϡ'), + (0x3E1, 'V'), + (0x3E2, 'M', u'ϣ'), + (0x3E3, 'V'), + (0x3E4, 'M', u'ϥ'), + (0x3E5, 'V'), + (0x3E6, 'M', u'ϧ'), + (0x3E7, 'V'), + (0x3E8, 'M', u'ϩ'), + (0x3E9, 'V'), + (0x3EA, 'M', u'ϫ'), + (0x3EB, 'V'), + (0x3EC, 'M', u'ϭ'), + (0x3ED, 'V'), + (0x3EE, 'M', u'ϯ'), + (0x3EF, 'V'), + (0x3F0, 'M', u'κ'), + (0x3F1, 'M', u'ρ'), + (0x3F2, 'M', u'σ'), + (0x3F3, 'V'), + (0x3F4, 'M', u'θ'), + (0x3F5, 'M', u'ε'), + (0x3F6, 'V'), + (0x3F7, 'M', u'ϸ'), + (0x3F8, 'V'), + (0x3F9, 'M', u'σ'), + (0x3FA, 'M', u'ϻ'), + (0x3FB, 'V'), + (0x3FD, 'M', u'ͻ'), + (0x3FE, 'M', u'ͼ'), + (0x3FF, 'M', u'ͽ'), + (0x400, 'M', u'ѐ'), + (0x401, 'M', u'ё'), + (0x402, 'M', u'ђ'), + ] + +def _seg_7(): + return [ + (0x403, 'M', u'ѓ'), + (0x404, 'M', u'є'), + (0x405, 'M', u'ѕ'), + (0x406, 'M', u'і'), + (0x407, 'M', u'ї'), + (0x408, 'M', u'ј'), + (0x409, 'M', u'љ'), + (0x40A, 'M', u'њ'), + (0x40B, 'M', u'ћ'), + (0x40C, 'M', u'ќ'), + (0x40D, 'M', u'ѝ'), + (0x40E, 'M', u'ў'), + (0x40F, 'M', u'џ'), + (0x410, 'M', u'а'), + (0x411, 'M', u'б'), + (0x412, 'M', u'в'), + (0x413, 'M', u'г'), + (0x414, 'M', u'д'), + (0x415, 'M', u'е'), + (0x416, 'M', u'ж'), + (0x417, 'M', u'з'), + (0x418, 'M', u'и'), + (0x419, 'M', u'й'), + (0x41A, 'M', u'к'), + (0x41B, 'M', u'л'), + (0x41C, 'M', u'м'), + (0x41D, 'M', u'н'), + (0x41E, 'M', u'о'), + (0x41F, 'M', u'п'), + (0x420, 'M', u'р'), + (0x421, 'M', u'с'), + (0x422, 'M', u'т'), + (0x423, 'M', u'у'), + (0x424, 'M', u'ф'), + (0x425, 'M', u'х'), + (0x426, 'M', u'ц'), + (0x427, 'M', u'ч'), + (0x428, 'M', u'ш'), + (0x429, 'M', u'щ'), + (0x42A, 'M', u'ъ'), + (0x42B, 'M', u'ы'), + (0x42C, 'M', u'ь'), + (0x42D, 'M', u'э'), + (0x42E, 'M', u'ю'), + (0x42F, 'M', u'я'), + (0x430, 'V'), + (0x460, 'M', u'ѡ'), + (0x461, 'V'), + (0x462, 'M', u'ѣ'), + (0x463, 'V'), + (0x464, 'M', u'ѥ'), + (0x465, 'V'), + (0x466, 'M', u'ѧ'), + (0x467, 'V'), + (0x468, 'M', u'ѩ'), + (0x469, 'V'), + (0x46A, 'M', u'ѫ'), + (0x46B, 'V'), + (0x46C, 'M', u'ѭ'), + (0x46D, 'V'), + (0x46E, 'M', u'ѯ'), + (0x46F, 'V'), + (0x470, 'M', u'ѱ'), + (0x471, 'V'), + (0x472, 'M', u'ѳ'), + (0x473, 'V'), + (0x474, 'M', u'ѵ'), + (0x475, 'V'), + (0x476, 'M', u'ѷ'), + (0x477, 'V'), + (0x478, 'M', u'ѹ'), + (0x479, 'V'), + (0x47A, 'M', u'ѻ'), + (0x47B, 'V'), + (0x47C, 'M', u'ѽ'), + (0x47D, 'V'), + (0x47E, 'M', u'ѿ'), + (0x47F, 'V'), + (0x480, 'M', u'ҁ'), + (0x481, 'V'), + (0x48A, 'M', u'ҋ'), + (0x48B, 'V'), + (0x48C, 'M', u'ҍ'), + (0x48D, 'V'), + (0x48E, 'M', u'ҏ'), + (0x48F, 'V'), + (0x490, 'M', u'ґ'), + (0x491, 'V'), + (0x492, 'M', u'ғ'), + (0x493, 'V'), + (0x494, 'M', u'ҕ'), + (0x495, 'V'), + (0x496, 'M', u'җ'), + (0x497, 'V'), + (0x498, 'M', u'ҙ'), + (0x499, 'V'), + (0x49A, 'M', u'қ'), + (0x49B, 'V'), + (0x49C, 'M', u'ҝ'), + (0x49D, 'V'), + ] + +def _seg_8(): + return [ + (0x49E, 'M', u'ҟ'), + (0x49F, 'V'), + (0x4A0, 'M', u'ҡ'), + (0x4A1, 'V'), + (0x4A2, 'M', u'ң'), + (0x4A3, 'V'), + (0x4A4, 'M', u'ҥ'), + (0x4A5, 'V'), + (0x4A6, 'M', u'ҧ'), + (0x4A7, 'V'), + (0x4A8, 'M', u'ҩ'), + (0x4A9, 'V'), + (0x4AA, 'M', u'ҫ'), + (0x4AB, 'V'), + (0x4AC, 'M', u'ҭ'), + (0x4AD, 'V'), + (0x4AE, 'M', u'ү'), + (0x4AF, 'V'), + (0x4B0, 'M', u'ұ'), + (0x4B1, 'V'), + (0x4B2, 'M', u'ҳ'), + (0x4B3, 'V'), + (0x4B4, 'M', u'ҵ'), + (0x4B5, 'V'), + (0x4B6, 'M', u'ҷ'), + (0x4B7, 'V'), + (0x4B8, 'M', u'ҹ'), + (0x4B9, 'V'), + (0x4BA, 'M', u'һ'), + (0x4BB, 'V'), + (0x4BC, 'M', u'ҽ'), + (0x4BD, 'V'), + (0x4BE, 'M', u'ҿ'), + (0x4BF, 'V'), + (0x4C0, 'X'), + (0x4C1, 'M', u'ӂ'), + (0x4C2, 'V'), + (0x4C3, 'M', u'ӄ'), + (0x4C4, 'V'), + (0x4C5, 'M', u'ӆ'), + (0x4C6, 'V'), + (0x4C7, 'M', u'ӈ'), + (0x4C8, 'V'), + (0x4C9, 'M', u'ӊ'), + (0x4CA, 'V'), + (0x4CB, 'M', u'ӌ'), + (0x4CC, 'V'), + (0x4CD, 'M', u'ӎ'), + (0x4CE, 'V'), + (0x4D0, 'M', u'ӑ'), + (0x4D1, 'V'), + (0x4D2, 'M', u'ӓ'), + (0x4D3, 'V'), + (0x4D4, 'M', u'ӕ'), + (0x4D5, 'V'), + (0x4D6, 'M', u'ӗ'), + (0x4D7, 'V'), + (0x4D8, 'M', u'ә'), + (0x4D9, 'V'), + (0x4DA, 'M', u'ӛ'), + (0x4DB, 'V'), + (0x4DC, 'M', u'ӝ'), + (0x4DD, 'V'), + (0x4DE, 'M', u'ӟ'), + (0x4DF, 'V'), + (0x4E0, 'M', u'ӡ'), + (0x4E1, 'V'), + (0x4E2, 'M', u'ӣ'), + (0x4E3, 'V'), + (0x4E4, 'M', u'ӥ'), + (0x4E5, 'V'), + (0x4E6, 'M', u'ӧ'), + (0x4E7, 'V'), + (0x4E8, 'M', u'ө'), + (0x4E9, 'V'), + (0x4EA, 'M', u'ӫ'), + (0x4EB, 'V'), + (0x4EC, 'M', u'ӭ'), + (0x4ED, 'V'), + (0x4EE, 'M', u'ӯ'), + (0x4EF, 'V'), + (0x4F0, 'M', u'ӱ'), + (0x4F1, 'V'), + (0x4F2, 'M', u'ӳ'), + (0x4F3, 'V'), + (0x4F4, 'M', u'ӵ'), + (0x4F5, 'V'), + (0x4F6, 'M', u'ӷ'), + (0x4F7, 'V'), + (0x4F8, 'M', u'ӹ'), + (0x4F9, 'V'), + (0x4FA, 'M', u'ӻ'), + (0x4FB, 'V'), + (0x4FC, 'M', u'ӽ'), + (0x4FD, 'V'), + (0x4FE, 'M', u'ӿ'), + (0x4FF, 'V'), + (0x500, 'M', u'ԁ'), + (0x501, 'V'), + (0x502, 'M', u'ԃ'), + ] + +def _seg_9(): + return [ + (0x503, 'V'), + (0x504, 'M', u'ԅ'), + (0x505, 'V'), + (0x506, 'M', u'ԇ'), + (0x507, 'V'), + (0x508, 'M', u'ԉ'), + (0x509, 'V'), + (0x50A, 'M', u'ԋ'), + (0x50B, 'V'), + (0x50C, 'M', u'ԍ'), + (0x50D, 'V'), + (0x50E, 'M', u'ԏ'), + (0x50F, 'V'), + (0x510, 'M', u'ԑ'), + (0x511, 'V'), + (0x512, 'M', u'ԓ'), + (0x513, 'V'), + (0x514, 'M', u'ԕ'), + (0x515, 'V'), + (0x516, 'M', u'ԗ'), + (0x517, 'V'), + (0x518, 'M', u'ԙ'), + (0x519, 'V'), + (0x51A, 'M', u'ԛ'), + (0x51B, 'V'), + (0x51C, 'M', u'ԝ'), + (0x51D, 'V'), + (0x51E, 'M', u'ԟ'), + (0x51F, 'V'), + (0x520, 'M', u'ԡ'), + (0x521, 'V'), + (0x522, 'M', u'ԣ'), + (0x523, 'V'), + (0x524, 'M', u'ԥ'), + (0x525, 'V'), + (0x526, 'M', u'ԧ'), + (0x527, 'V'), + (0x528, 'M', u'ԩ'), + (0x529, 'V'), + (0x52A, 'M', u'ԫ'), + (0x52B, 'V'), + (0x52C, 'M', u'ԭ'), + (0x52D, 'V'), + (0x52E, 'M', u'ԯ'), + (0x52F, 'V'), + (0x530, 'X'), + (0x531, 'M', u'ա'), + (0x532, 'M', u'բ'), + (0x533, 'M', u'գ'), + (0x534, 'M', u'դ'), + (0x535, 'M', u'ե'), + (0x536, 'M', u'զ'), + (0x537, 'M', u'է'), + (0x538, 'M', u'ը'), + (0x539, 'M', u'թ'), + (0x53A, 'M', u'ժ'), + (0x53B, 'M', u'ի'), + (0x53C, 'M', u'լ'), + (0x53D, 'M', u'խ'), + (0x53E, 'M', u'ծ'), + (0x53F, 'M', u'կ'), + (0x540, 'M', u'հ'), + (0x541, 'M', u'ձ'), + (0x542, 'M', u'ղ'), + (0x543, 'M', u'ճ'), + (0x544, 'M', u'մ'), + (0x545, 'M', u'յ'), + (0x546, 'M', u'ն'), + (0x547, 'M', u'շ'), + (0x548, 'M', u'ո'), + (0x549, 'M', u'չ'), + (0x54A, 'M', u'պ'), + (0x54B, 'M', u'ջ'), + (0x54C, 'M', u'ռ'), + (0x54D, 'M', u'ս'), + (0x54E, 'M', u'վ'), + (0x54F, 'M', u'տ'), + (0x550, 'M', u'ր'), + (0x551, 'M', u'ց'), + (0x552, 'M', u'ւ'), + (0x553, 'M', u'փ'), + (0x554, 'M', u'ք'), + (0x555, 'M', u'օ'), + (0x556, 'M', u'ֆ'), + (0x557, 'X'), + (0x559, 'V'), + (0x587, 'M', u'եւ'), + (0x588, 'V'), + (0x58B, 'X'), + (0x58D, 'V'), + (0x590, 'X'), + (0x591, 'V'), + (0x5C8, 'X'), + (0x5D0, 'V'), + (0x5EB, 'X'), + (0x5EF, 'V'), + (0x5F5, 'X'), + (0x606, 'V'), + (0x61C, 'X'), + (0x61E, 'V'), + ] + +def _seg_10(): + return [ + (0x675, 'M', u'اٴ'), + (0x676, 'M', u'وٴ'), + (0x677, 'M', u'ۇٴ'), + (0x678, 'M', u'يٴ'), + (0x679, 'V'), + (0x6DD, 'X'), + (0x6DE, 'V'), + (0x70E, 'X'), + (0x710, 'V'), + (0x74B, 'X'), + (0x74D, 'V'), + (0x7B2, 'X'), + (0x7C0, 'V'), + (0x7FB, 'X'), + (0x7FD, 'V'), + (0x82E, 'X'), + (0x830, 'V'), + (0x83F, 'X'), + (0x840, 'V'), + (0x85C, 'X'), + (0x85E, 'V'), + (0x85F, 'X'), + (0x860, 'V'), + (0x86B, 'X'), + (0x8A0, 'V'), + (0x8B5, 'X'), + (0x8B6, 'V'), + (0x8C8, 'X'), + (0x8D3, 'V'), + (0x8E2, 'X'), + (0x8E3, 'V'), + (0x958, 'M', u'क़'), + (0x959, 'M', u'ख़'), + (0x95A, 'M', u'ग़'), + (0x95B, 'M', u'ज़'), + (0x95C, 'M', u'ड़'), + (0x95D, 'M', u'ढ़'), + (0x95E, 'M', u'फ़'), + (0x95F, 'M', u'य़'), + (0x960, 'V'), + (0x984, 'X'), + (0x985, 'V'), + (0x98D, 'X'), + (0x98F, 'V'), + (0x991, 'X'), + (0x993, 'V'), + (0x9A9, 'X'), + (0x9AA, 'V'), + (0x9B1, 'X'), + (0x9B2, 'V'), + (0x9B3, 'X'), + (0x9B6, 'V'), + (0x9BA, 'X'), + (0x9BC, 'V'), + (0x9C5, 'X'), + (0x9C7, 'V'), + (0x9C9, 'X'), + (0x9CB, 'V'), + (0x9CF, 'X'), + (0x9D7, 'V'), + (0x9D8, 'X'), + (0x9DC, 'M', u'ড়'), + (0x9DD, 'M', u'ঢ়'), + (0x9DE, 'X'), + (0x9DF, 'M', u'য়'), + (0x9E0, 'V'), + (0x9E4, 'X'), + (0x9E6, 'V'), + (0x9FF, 'X'), + (0xA01, 'V'), + (0xA04, 'X'), + (0xA05, 'V'), + (0xA0B, 'X'), + (0xA0F, 'V'), + (0xA11, 'X'), + (0xA13, 'V'), + (0xA29, 'X'), + (0xA2A, 'V'), + (0xA31, 'X'), + (0xA32, 'V'), + (0xA33, 'M', u'ਲ਼'), + (0xA34, 'X'), + (0xA35, 'V'), + (0xA36, 'M', u'ਸ਼'), + (0xA37, 'X'), + (0xA38, 'V'), + (0xA3A, 'X'), + (0xA3C, 'V'), + (0xA3D, 'X'), + (0xA3E, 'V'), + (0xA43, 'X'), + (0xA47, 'V'), + (0xA49, 'X'), + (0xA4B, 'V'), + (0xA4E, 'X'), + (0xA51, 'V'), + (0xA52, 'X'), + (0xA59, 'M', u'ਖ਼'), + (0xA5A, 'M', u'ਗ਼'), + (0xA5B, 'M', u'ਜ਼'), + ] + +def _seg_11(): + return [ + (0xA5C, 'V'), + (0xA5D, 'X'), + (0xA5E, 'M', u'ਫ਼'), + (0xA5F, 'X'), + (0xA66, 'V'), + (0xA77, 'X'), + (0xA81, 'V'), + (0xA84, 'X'), + (0xA85, 'V'), + (0xA8E, 'X'), + (0xA8F, 'V'), + (0xA92, 'X'), + (0xA93, 'V'), + (0xAA9, 'X'), + (0xAAA, 'V'), + (0xAB1, 'X'), + (0xAB2, 'V'), + (0xAB4, 'X'), + (0xAB5, 'V'), + (0xABA, 'X'), + (0xABC, 'V'), + (0xAC6, 'X'), + (0xAC7, 'V'), + (0xACA, 'X'), + (0xACB, 'V'), + (0xACE, 'X'), + (0xAD0, 'V'), + (0xAD1, 'X'), + (0xAE0, 'V'), + (0xAE4, 'X'), + (0xAE6, 'V'), + (0xAF2, 'X'), + (0xAF9, 'V'), + (0xB00, 'X'), + (0xB01, 'V'), + (0xB04, 'X'), + (0xB05, 'V'), + (0xB0D, 'X'), + (0xB0F, 'V'), + (0xB11, 'X'), + (0xB13, 'V'), + (0xB29, 'X'), + (0xB2A, 'V'), + (0xB31, 'X'), + (0xB32, 'V'), + (0xB34, 'X'), + (0xB35, 'V'), + (0xB3A, 'X'), + (0xB3C, 'V'), + (0xB45, 'X'), + (0xB47, 'V'), + (0xB49, 'X'), + (0xB4B, 'V'), + (0xB4E, 'X'), + (0xB55, 'V'), + (0xB58, 'X'), + (0xB5C, 'M', u'ଡ଼'), + (0xB5D, 'M', u'ଢ଼'), + (0xB5E, 'X'), + (0xB5F, 'V'), + (0xB64, 'X'), + (0xB66, 'V'), + (0xB78, 'X'), + (0xB82, 'V'), + (0xB84, 'X'), + (0xB85, 'V'), + (0xB8B, 'X'), + (0xB8E, 'V'), + (0xB91, 'X'), + (0xB92, 'V'), + (0xB96, 'X'), + (0xB99, 'V'), + (0xB9B, 'X'), + (0xB9C, 'V'), + (0xB9D, 'X'), + (0xB9E, 'V'), + (0xBA0, 'X'), + (0xBA3, 'V'), + (0xBA5, 'X'), + (0xBA8, 'V'), + (0xBAB, 'X'), + (0xBAE, 'V'), + (0xBBA, 'X'), + (0xBBE, 'V'), + (0xBC3, 'X'), + (0xBC6, 'V'), + (0xBC9, 'X'), + (0xBCA, 'V'), + (0xBCE, 'X'), + (0xBD0, 'V'), + (0xBD1, 'X'), + (0xBD7, 'V'), + (0xBD8, 'X'), + (0xBE6, 'V'), + (0xBFB, 'X'), + (0xC00, 'V'), + (0xC0D, 'X'), + (0xC0E, 'V'), + (0xC11, 'X'), + (0xC12, 'V'), + ] + +def _seg_12(): + return [ + (0xC29, 'X'), + (0xC2A, 'V'), + (0xC3A, 'X'), + (0xC3D, 'V'), + (0xC45, 'X'), + (0xC46, 'V'), + (0xC49, 'X'), + (0xC4A, 'V'), + (0xC4E, 'X'), + (0xC55, 'V'), + (0xC57, 'X'), + (0xC58, 'V'), + (0xC5B, 'X'), + (0xC60, 'V'), + (0xC64, 'X'), + (0xC66, 'V'), + (0xC70, 'X'), + (0xC77, 'V'), + (0xC8D, 'X'), + (0xC8E, 'V'), + (0xC91, 'X'), + (0xC92, 'V'), + (0xCA9, 'X'), + (0xCAA, 'V'), + (0xCB4, 'X'), + (0xCB5, 'V'), + (0xCBA, 'X'), + (0xCBC, 'V'), + (0xCC5, 'X'), + (0xCC6, 'V'), + (0xCC9, 'X'), + (0xCCA, 'V'), + (0xCCE, 'X'), + (0xCD5, 'V'), + (0xCD7, 'X'), + (0xCDE, 'V'), + (0xCDF, 'X'), + (0xCE0, 'V'), + (0xCE4, 'X'), + (0xCE6, 'V'), + (0xCF0, 'X'), + (0xCF1, 'V'), + (0xCF3, 'X'), + (0xD00, 'V'), + (0xD0D, 'X'), + (0xD0E, 'V'), + (0xD11, 'X'), + (0xD12, 'V'), + (0xD45, 'X'), + (0xD46, 'V'), + (0xD49, 'X'), + (0xD4A, 'V'), + (0xD50, 'X'), + (0xD54, 'V'), + (0xD64, 'X'), + (0xD66, 'V'), + (0xD80, 'X'), + (0xD81, 'V'), + (0xD84, 'X'), + (0xD85, 'V'), + (0xD97, 'X'), + (0xD9A, 'V'), + (0xDB2, 'X'), + (0xDB3, 'V'), + (0xDBC, 'X'), + (0xDBD, 'V'), + (0xDBE, 'X'), + (0xDC0, 'V'), + (0xDC7, 'X'), + (0xDCA, 'V'), + (0xDCB, 'X'), + (0xDCF, 'V'), + (0xDD5, 'X'), + (0xDD6, 'V'), + (0xDD7, 'X'), + (0xDD8, 'V'), + (0xDE0, 'X'), + (0xDE6, 'V'), + (0xDF0, 'X'), + (0xDF2, 'V'), + (0xDF5, 'X'), + (0xE01, 'V'), + (0xE33, 'M', u'ํา'), + (0xE34, 'V'), + (0xE3B, 'X'), + (0xE3F, 'V'), + (0xE5C, 'X'), + (0xE81, 'V'), + (0xE83, 'X'), + (0xE84, 'V'), + (0xE85, 'X'), + (0xE86, 'V'), + (0xE8B, 'X'), + (0xE8C, 'V'), + (0xEA4, 'X'), + (0xEA5, 'V'), + (0xEA6, 'X'), + (0xEA7, 'V'), + (0xEB3, 'M', u'ໍາ'), + (0xEB4, 'V'), + ] + +def _seg_13(): + return [ + (0xEBE, 'X'), + (0xEC0, 'V'), + (0xEC5, 'X'), + (0xEC6, 'V'), + (0xEC7, 'X'), + (0xEC8, 'V'), + (0xECE, 'X'), + (0xED0, 'V'), + (0xEDA, 'X'), + (0xEDC, 'M', u'ຫນ'), + (0xEDD, 'M', u'ຫມ'), + (0xEDE, 'V'), + (0xEE0, 'X'), + (0xF00, 'V'), + (0xF0C, 'M', u'་'), + (0xF0D, 'V'), + (0xF43, 'M', u'གྷ'), + (0xF44, 'V'), + (0xF48, 'X'), + (0xF49, 'V'), + (0xF4D, 'M', u'ཌྷ'), + (0xF4E, 'V'), + (0xF52, 'M', u'དྷ'), + (0xF53, 'V'), + (0xF57, 'M', u'བྷ'), + (0xF58, 'V'), + (0xF5C, 'M', u'ཛྷ'), + (0xF5D, 'V'), + (0xF69, 'M', u'ཀྵ'), + (0xF6A, 'V'), + (0xF6D, 'X'), + (0xF71, 'V'), + (0xF73, 'M', u'ཱི'), + (0xF74, 'V'), + (0xF75, 'M', u'ཱུ'), + (0xF76, 'M', u'ྲྀ'), + (0xF77, 'M', u'ྲཱྀ'), + (0xF78, 'M', u'ླྀ'), + (0xF79, 'M', u'ླཱྀ'), + (0xF7A, 'V'), + (0xF81, 'M', u'ཱྀ'), + (0xF82, 'V'), + (0xF93, 'M', u'ྒྷ'), + (0xF94, 'V'), + (0xF98, 'X'), + (0xF99, 'V'), + (0xF9D, 'M', u'ྜྷ'), + (0xF9E, 'V'), + (0xFA2, 'M', u'ྡྷ'), + (0xFA3, 'V'), + (0xFA7, 'M', u'ྦྷ'), + (0xFA8, 'V'), + (0xFAC, 'M', u'ྫྷ'), + (0xFAD, 'V'), + (0xFB9, 'M', u'ྐྵ'), + (0xFBA, 'V'), + (0xFBD, 'X'), + (0xFBE, 'V'), + (0xFCD, 'X'), + (0xFCE, 'V'), + (0xFDB, 'X'), + (0x1000, 'V'), + (0x10A0, 'X'), + (0x10C7, 'M', u'ⴧ'), + (0x10C8, 'X'), + (0x10CD, 'M', u'ⴭ'), + (0x10CE, 'X'), + (0x10D0, 'V'), + (0x10FC, 'M', u'ნ'), + (0x10FD, 'V'), + (0x115F, 'X'), + (0x1161, 'V'), + (0x1249, 'X'), + (0x124A, 'V'), + (0x124E, 'X'), + (0x1250, 'V'), + (0x1257, 'X'), + (0x1258, 'V'), + (0x1259, 'X'), + (0x125A, 'V'), + (0x125E, 'X'), + (0x1260, 'V'), + (0x1289, 'X'), + (0x128A, 'V'), + (0x128E, 'X'), + (0x1290, 'V'), + (0x12B1, 'X'), + (0x12B2, 'V'), + (0x12B6, 'X'), + (0x12B8, 'V'), + (0x12BF, 'X'), + (0x12C0, 'V'), + (0x12C1, 'X'), + (0x12C2, 'V'), + (0x12C6, 'X'), + (0x12C8, 'V'), + (0x12D7, 'X'), + (0x12D8, 'V'), + (0x1311, 'X'), + (0x1312, 'V'), + ] + +def _seg_14(): + return [ + (0x1316, 'X'), + (0x1318, 'V'), + (0x135B, 'X'), + (0x135D, 'V'), + (0x137D, 'X'), + (0x1380, 'V'), + (0x139A, 'X'), + (0x13A0, 'V'), + (0x13F6, 'X'), + (0x13F8, 'M', u'Ᏸ'), + (0x13F9, 'M', u'Ᏹ'), + (0x13FA, 'M', u'Ᏺ'), + (0x13FB, 'M', u'Ᏻ'), + (0x13FC, 'M', u'Ᏼ'), + (0x13FD, 'M', u'Ᏽ'), + (0x13FE, 'X'), + (0x1400, 'V'), + (0x1680, 'X'), + (0x1681, 'V'), + (0x169D, 'X'), + (0x16A0, 'V'), + (0x16F9, 'X'), + (0x1700, 'V'), + (0x170D, 'X'), + (0x170E, 'V'), + (0x1715, 'X'), + (0x1720, 'V'), + (0x1737, 'X'), + (0x1740, 'V'), + (0x1754, 'X'), + (0x1760, 'V'), + (0x176D, 'X'), + (0x176E, 'V'), + (0x1771, 'X'), + (0x1772, 'V'), + (0x1774, 'X'), + (0x1780, 'V'), + (0x17B4, 'X'), + (0x17B6, 'V'), + (0x17DE, 'X'), + (0x17E0, 'V'), + (0x17EA, 'X'), + (0x17F0, 'V'), + (0x17FA, 'X'), + (0x1800, 'V'), + (0x1806, 'X'), + (0x1807, 'V'), + (0x180B, 'I'), + (0x180E, 'X'), + (0x1810, 'V'), + (0x181A, 'X'), + (0x1820, 'V'), + (0x1879, 'X'), + (0x1880, 'V'), + (0x18AB, 'X'), + (0x18B0, 'V'), + (0x18F6, 'X'), + (0x1900, 'V'), + (0x191F, 'X'), + (0x1920, 'V'), + (0x192C, 'X'), + (0x1930, 'V'), + (0x193C, 'X'), + (0x1940, 'V'), + (0x1941, 'X'), + (0x1944, 'V'), + (0x196E, 'X'), + (0x1970, 'V'), + (0x1975, 'X'), + (0x1980, 'V'), + (0x19AC, 'X'), + (0x19B0, 'V'), + (0x19CA, 'X'), + (0x19D0, 'V'), + (0x19DB, 'X'), + (0x19DE, 'V'), + (0x1A1C, 'X'), + (0x1A1E, 'V'), + (0x1A5F, 'X'), + (0x1A60, 'V'), + (0x1A7D, 'X'), + (0x1A7F, 'V'), + (0x1A8A, 'X'), + (0x1A90, 'V'), + (0x1A9A, 'X'), + (0x1AA0, 'V'), + (0x1AAE, 'X'), + (0x1AB0, 'V'), + (0x1AC1, 'X'), + (0x1B00, 'V'), + (0x1B4C, 'X'), + (0x1B50, 'V'), + (0x1B7D, 'X'), + (0x1B80, 'V'), + (0x1BF4, 'X'), + (0x1BFC, 'V'), + (0x1C38, 'X'), + (0x1C3B, 'V'), + (0x1C4A, 'X'), + (0x1C4D, 'V'), + ] + +def _seg_15(): + return [ + (0x1C80, 'M', u'в'), + (0x1C81, 'M', u'д'), + (0x1C82, 'M', u'о'), + (0x1C83, 'M', u'с'), + (0x1C84, 'M', u'т'), + (0x1C86, 'M', u'ъ'), + (0x1C87, 'M', u'ѣ'), + (0x1C88, 'M', u'ꙋ'), + (0x1C89, 'X'), + (0x1C90, 'M', u'ა'), + (0x1C91, 'M', u'ბ'), + (0x1C92, 'M', u'გ'), + (0x1C93, 'M', u'დ'), + (0x1C94, 'M', u'ე'), + (0x1C95, 'M', u'ვ'), + (0x1C96, 'M', u'ზ'), + (0x1C97, 'M', u'თ'), + (0x1C98, 'M', u'ი'), + (0x1C99, 'M', u'კ'), + (0x1C9A, 'M', u'ლ'), + (0x1C9B, 'M', u'მ'), + (0x1C9C, 'M', u'ნ'), + (0x1C9D, 'M', u'ო'), + (0x1C9E, 'M', u'პ'), + (0x1C9F, 'M', u'ჟ'), + (0x1CA0, 'M', u'რ'), + (0x1CA1, 'M', u'ს'), + (0x1CA2, 'M', u'ტ'), + (0x1CA3, 'M', u'უ'), + (0x1CA4, 'M', u'ფ'), + (0x1CA5, 'M', u'ქ'), + (0x1CA6, 'M', u'ღ'), + (0x1CA7, 'M', u'ყ'), + (0x1CA8, 'M', u'შ'), + (0x1CA9, 'M', u'ჩ'), + (0x1CAA, 'M', u'ც'), + (0x1CAB, 'M', u'ძ'), + (0x1CAC, 'M', u'წ'), + (0x1CAD, 'M', u'ჭ'), + (0x1CAE, 'M', u'ხ'), + (0x1CAF, 'M', u'ჯ'), + (0x1CB0, 'M', u'ჰ'), + (0x1CB1, 'M', u'ჱ'), + (0x1CB2, 'M', u'ჲ'), + (0x1CB3, 'M', u'ჳ'), + (0x1CB4, 'M', u'ჴ'), + (0x1CB5, 'M', u'ჵ'), + (0x1CB6, 'M', u'ჶ'), + (0x1CB7, 'M', u'ჷ'), + (0x1CB8, 'M', u'ჸ'), + (0x1CB9, 'M', u'ჹ'), + (0x1CBA, 'M', u'ჺ'), + (0x1CBB, 'X'), + (0x1CBD, 'M', u'ჽ'), + (0x1CBE, 'M', u'ჾ'), + (0x1CBF, 'M', u'ჿ'), + (0x1CC0, 'V'), + (0x1CC8, 'X'), + (0x1CD0, 'V'), + (0x1CFB, 'X'), + (0x1D00, 'V'), + (0x1D2C, 'M', u'a'), + (0x1D2D, 'M', u'æ'), + (0x1D2E, 'M', u'b'), + (0x1D2F, 'V'), + (0x1D30, 'M', u'd'), + (0x1D31, 'M', u'e'), + (0x1D32, 'M', u'ǝ'), + (0x1D33, 'M', u'g'), + (0x1D34, 'M', u'h'), + (0x1D35, 'M', u'i'), + (0x1D36, 'M', u'j'), + (0x1D37, 'M', u'k'), + (0x1D38, 'M', u'l'), + (0x1D39, 'M', u'm'), + (0x1D3A, 'M', u'n'), + (0x1D3B, 'V'), + (0x1D3C, 'M', u'o'), + (0x1D3D, 'M', u'ȣ'), + (0x1D3E, 'M', u'p'), + (0x1D3F, 'M', u'r'), + (0x1D40, 'M', u't'), + (0x1D41, 'M', u'u'), + (0x1D42, 'M', u'w'), + (0x1D43, 'M', u'a'), + (0x1D44, 'M', u'ɐ'), + (0x1D45, 'M', u'ɑ'), + (0x1D46, 'M', u'ᴂ'), + (0x1D47, 'M', u'b'), + (0x1D48, 'M', u'd'), + (0x1D49, 'M', u'e'), + (0x1D4A, 'M', u'ə'), + (0x1D4B, 'M', u'ɛ'), + (0x1D4C, 'M', u'ɜ'), + (0x1D4D, 'M', u'g'), + (0x1D4E, 'V'), + (0x1D4F, 'M', u'k'), + (0x1D50, 'M', u'm'), + (0x1D51, 'M', u'ŋ'), + (0x1D52, 'M', u'o'), + ] + +def _seg_16(): + return [ + (0x1D53, 'M', u'ɔ'), + (0x1D54, 'M', u'ᴖ'), + (0x1D55, 'M', u'ᴗ'), + (0x1D56, 'M', u'p'), + (0x1D57, 'M', u't'), + (0x1D58, 'M', u'u'), + (0x1D59, 'M', u'ᴝ'), + (0x1D5A, 'M', u'ɯ'), + (0x1D5B, 'M', u'v'), + (0x1D5C, 'M', u'ᴥ'), + (0x1D5D, 'M', u'β'), + (0x1D5E, 'M', u'γ'), + (0x1D5F, 'M', u'δ'), + (0x1D60, 'M', u'φ'), + (0x1D61, 'M', u'χ'), + (0x1D62, 'M', u'i'), + (0x1D63, 'M', u'r'), + (0x1D64, 'M', u'u'), + (0x1D65, 'M', u'v'), + (0x1D66, 'M', u'β'), + (0x1D67, 'M', u'γ'), + (0x1D68, 'M', u'ρ'), + (0x1D69, 'M', u'φ'), + (0x1D6A, 'M', u'χ'), + (0x1D6B, 'V'), + (0x1D78, 'M', u'н'), + (0x1D79, 'V'), + (0x1D9B, 'M', u'ɒ'), + (0x1D9C, 'M', u'c'), + (0x1D9D, 'M', u'ɕ'), + (0x1D9E, 'M', u'ð'), + (0x1D9F, 'M', u'ɜ'), + (0x1DA0, 'M', u'f'), + (0x1DA1, 'M', u'ɟ'), + (0x1DA2, 'M', u'ɡ'), + (0x1DA3, 'M', u'ɥ'), + (0x1DA4, 'M', u'ɨ'), + (0x1DA5, 'M', u'ɩ'), + (0x1DA6, 'M', u'ɪ'), + (0x1DA7, 'M', u'ᵻ'), + (0x1DA8, 'M', u'ʝ'), + (0x1DA9, 'M', u'ɭ'), + (0x1DAA, 'M', u'ᶅ'), + (0x1DAB, 'M', u'ʟ'), + (0x1DAC, 'M', u'ɱ'), + (0x1DAD, 'M', u'ɰ'), + (0x1DAE, 'M', u'ɲ'), + (0x1DAF, 'M', u'ɳ'), + (0x1DB0, 'M', u'ɴ'), + (0x1DB1, 'M', u'ɵ'), + (0x1DB2, 'M', u'ɸ'), + (0x1DB3, 'M', u'ʂ'), + (0x1DB4, 'M', u'ʃ'), + (0x1DB5, 'M', u'ƫ'), + (0x1DB6, 'M', u'ʉ'), + (0x1DB7, 'M', u'ʊ'), + (0x1DB8, 'M', u'ᴜ'), + (0x1DB9, 'M', u'ʋ'), + (0x1DBA, 'M', u'ʌ'), + (0x1DBB, 'M', u'z'), + (0x1DBC, 'M', u'ʐ'), + (0x1DBD, 'M', u'ʑ'), + (0x1DBE, 'M', u'ʒ'), + (0x1DBF, 'M', u'θ'), + (0x1DC0, 'V'), + (0x1DFA, 'X'), + (0x1DFB, 'V'), + (0x1E00, 'M', u'ḁ'), + (0x1E01, 'V'), + (0x1E02, 'M', u'ḃ'), + (0x1E03, 'V'), + (0x1E04, 'M', u'ḅ'), + (0x1E05, 'V'), + (0x1E06, 'M', u'ḇ'), + (0x1E07, 'V'), + (0x1E08, 'M', u'ḉ'), + (0x1E09, 'V'), + (0x1E0A, 'M', u'ḋ'), + (0x1E0B, 'V'), + (0x1E0C, 'M', u'ḍ'), + (0x1E0D, 'V'), + (0x1E0E, 'M', u'ḏ'), + (0x1E0F, 'V'), + (0x1E10, 'M', u'ḑ'), + (0x1E11, 'V'), + (0x1E12, 'M', u'ḓ'), + (0x1E13, 'V'), + (0x1E14, 'M', u'ḕ'), + (0x1E15, 'V'), + (0x1E16, 'M', u'ḗ'), + (0x1E17, 'V'), + (0x1E18, 'M', u'ḙ'), + (0x1E19, 'V'), + (0x1E1A, 'M', u'ḛ'), + (0x1E1B, 'V'), + (0x1E1C, 'M', u'ḝ'), + (0x1E1D, 'V'), + (0x1E1E, 'M', u'ḟ'), + (0x1E1F, 'V'), + (0x1E20, 'M', u'ḡ'), + ] + +def _seg_17(): + return [ + (0x1E21, 'V'), + (0x1E22, 'M', u'ḣ'), + (0x1E23, 'V'), + (0x1E24, 'M', u'ḥ'), + (0x1E25, 'V'), + (0x1E26, 'M', u'ḧ'), + (0x1E27, 'V'), + (0x1E28, 'M', u'ḩ'), + (0x1E29, 'V'), + (0x1E2A, 'M', u'ḫ'), + (0x1E2B, 'V'), + (0x1E2C, 'M', u'ḭ'), + (0x1E2D, 'V'), + (0x1E2E, 'M', u'ḯ'), + (0x1E2F, 'V'), + (0x1E30, 'M', u'ḱ'), + (0x1E31, 'V'), + (0x1E32, 'M', u'ḳ'), + (0x1E33, 'V'), + (0x1E34, 'M', u'ḵ'), + (0x1E35, 'V'), + (0x1E36, 'M', u'ḷ'), + (0x1E37, 'V'), + (0x1E38, 'M', u'ḹ'), + (0x1E39, 'V'), + (0x1E3A, 'M', u'ḻ'), + (0x1E3B, 'V'), + (0x1E3C, 'M', u'ḽ'), + (0x1E3D, 'V'), + (0x1E3E, 'M', u'ḿ'), + (0x1E3F, 'V'), + (0x1E40, 'M', u'ṁ'), + (0x1E41, 'V'), + (0x1E42, 'M', u'ṃ'), + (0x1E43, 'V'), + (0x1E44, 'M', u'ṅ'), + (0x1E45, 'V'), + (0x1E46, 'M', u'ṇ'), + (0x1E47, 'V'), + (0x1E48, 'M', u'ṉ'), + (0x1E49, 'V'), + (0x1E4A, 'M', u'ṋ'), + (0x1E4B, 'V'), + (0x1E4C, 'M', u'ṍ'), + (0x1E4D, 'V'), + (0x1E4E, 'M', u'ṏ'), + (0x1E4F, 'V'), + (0x1E50, 'M', u'ṑ'), + (0x1E51, 'V'), + (0x1E52, 'M', u'ṓ'), + (0x1E53, 'V'), + (0x1E54, 'M', u'ṕ'), + (0x1E55, 'V'), + (0x1E56, 'M', u'ṗ'), + (0x1E57, 'V'), + (0x1E58, 'M', u'ṙ'), + (0x1E59, 'V'), + (0x1E5A, 'M', u'ṛ'), + (0x1E5B, 'V'), + (0x1E5C, 'M', u'ṝ'), + (0x1E5D, 'V'), + (0x1E5E, 'M', u'ṟ'), + (0x1E5F, 'V'), + (0x1E60, 'M', u'ṡ'), + (0x1E61, 'V'), + (0x1E62, 'M', u'ṣ'), + (0x1E63, 'V'), + (0x1E64, 'M', u'ṥ'), + (0x1E65, 'V'), + (0x1E66, 'M', u'ṧ'), + (0x1E67, 'V'), + (0x1E68, 'M', u'ṩ'), + (0x1E69, 'V'), + (0x1E6A, 'M', u'ṫ'), + (0x1E6B, 'V'), + (0x1E6C, 'M', u'ṭ'), + (0x1E6D, 'V'), + (0x1E6E, 'M', u'ṯ'), + (0x1E6F, 'V'), + (0x1E70, 'M', u'ṱ'), + (0x1E71, 'V'), + (0x1E72, 'M', u'ṳ'), + (0x1E73, 'V'), + (0x1E74, 'M', u'ṵ'), + (0x1E75, 'V'), + (0x1E76, 'M', u'ṷ'), + (0x1E77, 'V'), + (0x1E78, 'M', u'ṹ'), + (0x1E79, 'V'), + (0x1E7A, 'M', u'ṻ'), + (0x1E7B, 'V'), + (0x1E7C, 'M', u'ṽ'), + (0x1E7D, 'V'), + (0x1E7E, 'M', u'ṿ'), + (0x1E7F, 'V'), + (0x1E80, 'M', u'ẁ'), + (0x1E81, 'V'), + (0x1E82, 'M', u'ẃ'), + (0x1E83, 'V'), + (0x1E84, 'M', u'ẅ'), + ] + +def _seg_18(): + return [ + (0x1E85, 'V'), + (0x1E86, 'M', u'ẇ'), + (0x1E87, 'V'), + (0x1E88, 'M', u'ẉ'), + (0x1E89, 'V'), + (0x1E8A, 'M', u'ẋ'), + (0x1E8B, 'V'), + (0x1E8C, 'M', u'ẍ'), + (0x1E8D, 'V'), + (0x1E8E, 'M', u'ẏ'), + (0x1E8F, 'V'), + (0x1E90, 'M', u'ẑ'), + (0x1E91, 'V'), + (0x1E92, 'M', u'ẓ'), + (0x1E93, 'V'), + (0x1E94, 'M', u'ẕ'), + (0x1E95, 'V'), + (0x1E9A, 'M', u'aʾ'), + (0x1E9B, 'M', u'ṡ'), + (0x1E9C, 'V'), + (0x1E9E, 'M', u'ss'), + (0x1E9F, 'V'), + (0x1EA0, 'M', u'ạ'), + (0x1EA1, 'V'), + (0x1EA2, 'M', u'ả'), + (0x1EA3, 'V'), + (0x1EA4, 'M', u'ấ'), + (0x1EA5, 'V'), + (0x1EA6, 'M', u'ầ'), + (0x1EA7, 'V'), + (0x1EA8, 'M', u'ẩ'), + (0x1EA9, 'V'), + (0x1EAA, 'M', u'ẫ'), + (0x1EAB, 'V'), + (0x1EAC, 'M', u'ậ'), + (0x1EAD, 'V'), + (0x1EAE, 'M', u'ắ'), + (0x1EAF, 'V'), + (0x1EB0, 'M', u'ằ'), + (0x1EB1, 'V'), + (0x1EB2, 'M', u'ẳ'), + (0x1EB3, 'V'), + (0x1EB4, 'M', u'ẵ'), + (0x1EB5, 'V'), + (0x1EB6, 'M', u'ặ'), + (0x1EB7, 'V'), + (0x1EB8, 'M', u'ẹ'), + (0x1EB9, 'V'), + (0x1EBA, 'M', u'ẻ'), + (0x1EBB, 'V'), + (0x1EBC, 'M', u'ẽ'), + (0x1EBD, 'V'), + (0x1EBE, 'M', u'ế'), + (0x1EBF, 'V'), + (0x1EC0, 'M', u'ề'), + (0x1EC1, 'V'), + (0x1EC2, 'M', u'ể'), + (0x1EC3, 'V'), + (0x1EC4, 'M', u'ễ'), + (0x1EC5, 'V'), + (0x1EC6, 'M', u'ệ'), + (0x1EC7, 'V'), + (0x1EC8, 'M', u'ỉ'), + (0x1EC9, 'V'), + (0x1ECA, 'M', u'ị'), + (0x1ECB, 'V'), + (0x1ECC, 'M', u'ọ'), + (0x1ECD, 'V'), + (0x1ECE, 'M', u'ỏ'), + (0x1ECF, 'V'), + (0x1ED0, 'M', u'ố'), + (0x1ED1, 'V'), + (0x1ED2, 'M', u'ồ'), + (0x1ED3, 'V'), + (0x1ED4, 'M', u'ổ'), + (0x1ED5, 'V'), + (0x1ED6, 'M', u'ỗ'), + (0x1ED7, 'V'), + (0x1ED8, 'M', u'ộ'), + (0x1ED9, 'V'), + (0x1EDA, 'M', u'ớ'), + (0x1EDB, 'V'), + (0x1EDC, 'M', u'ờ'), + (0x1EDD, 'V'), + (0x1EDE, 'M', u'ở'), + (0x1EDF, 'V'), + (0x1EE0, 'M', u'ỡ'), + (0x1EE1, 'V'), + (0x1EE2, 'M', u'ợ'), + (0x1EE3, 'V'), + (0x1EE4, 'M', u'ụ'), + (0x1EE5, 'V'), + (0x1EE6, 'M', u'ủ'), + (0x1EE7, 'V'), + (0x1EE8, 'M', u'ứ'), + (0x1EE9, 'V'), + (0x1EEA, 'M', u'ừ'), + (0x1EEB, 'V'), + (0x1EEC, 'M', u'ử'), + (0x1EED, 'V'), + ] + +def _seg_19(): + return [ + (0x1EEE, 'M', u'ữ'), + (0x1EEF, 'V'), + (0x1EF0, 'M', u'ự'), + (0x1EF1, 'V'), + (0x1EF2, 'M', u'ỳ'), + (0x1EF3, 'V'), + (0x1EF4, 'M', u'ỵ'), + (0x1EF5, 'V'), + (0x1EF6, 'M', u'ỷ'), + (0x1EF7, 'V'), + (0x1EF8, 'M', u'ỹ'), + (0x1EF9, 'V'), + (0x1EFA, 'M', u'ỻ'), + (0x1EFB, 'V'), + (0x1EFC, 'M', u'ỽ'), + (0x1EFD, 'V'), + (0x1EFE, 'M', u'ỿ'), + (0x1EFF, 'V'), + (0x1F08, 'M', u'ἀ'), + (0x1F09, 'M', u'ἁ'), + (0x1F0A, 'M', u'ἂ'), + (0x1F0B, 'M', u'ἃ'), + (0x1F0C, 'M', u'ἄ'), + (0x1F0D, 'M', u'ἅ'), + (0x1F0E, 'M', u'ἆ'), + (0x1F0F, 'M', u'ἇ'), + (0x1F10, 'V'), + (0x1F16, 'X'), + (0x1F18, 'M', u'ἐ'), + (0x1F19, 'M', u'ἑ'), + (0x1F1A, 'M', u'ἒ'), + (0x1F1B, 'M', u'ἓ'), + (0x1F1C, 'M', u'ἔ'), + (0x1F1D, 'M', u'ἕ'), + (0x1F1E, 'X'), + (0x1F20, 'V'), + (0x1F28, 'M', u'ἠ'), + (0x1F29, 'M', u'ἡ'), + (0x1F2A, 'M', u'ἢ'), + (0x1F2B, 'M', u'ἣ'), + (0x1F2C, 'M', u'ἤ'), + (0x1F2D, 'M', u'ἥ'), + (0x1F2E, 'M', u'ἦ'), + (0x1F2F, 'M', u'ἧ'), + (0x1F30, 'V'), + (0x1F38, 'M', u'ἰ'), + (0x1F39, 'M', u'ἱ'), + (0x1F3A, 'M', u'ἲ'), + (0x1F3B, 'M', u'ἳ'), + (0x1F3C, 'M', u'ἴ'), + (0x1F3D, 'M', u'ἵ'), + (0x1F3E, 'M', u'ἶ'), + (0x1F3F, 'M', u'ἷ'), + (0x1F40, 'V'), + (0x1F46, 'X'), + (0x1F48, 'M', u'ὀ'), + (0x1F49, 'M', u'ὁ'), + (0x1F4A, 'M', u'ὂ'), + (0x1F4B, 'M', u'ὃ'), + (0x1F4C, 'M', u'ὄ'), + (0x1F4D, 'M', u'ὅ'), + (0x1F4E, 'X'), + (0x1F50, 'V'), + (0x1F58, 'X'), + (0x1F59, 'M', u'ὑ'), + (0x1F5A, 'X'), + (0x1F5B, 'M', u'ὓ'), + (0x1F5C, 'X'), + (0x1F5D, 'M', u'ὕ'), + (0x1F5E, 'X'), + (0x1F5F, 'M', u'ὗ'), + (0x1F60, 'V'), + (0x1F68, 'M', u'ὠ'), + (0x1F69, 'M', u'ὡ'), + (0x1F6A, 'M', u'ὢ'), + (0x1F6B, 'M', u'ὣ'), + (0x1F6C, 'M', u'ὤ'), + (0x1F6D, 'M', u'ὥ'), + (0x1F6E, 'M', u'ὦ'), + (0x1F6F, 'M', u'ὧ'), + (0x1F70, 'V'), + (0x1F71, 'M', u'ά'), + (0x1F72, 'V'), + (0x1F73, 'M', u'έ'), + (0x1F74, 'V'), + (0x1F75, 'M', u'ή'), + (0x1F76, 'V'), + (0x1F77, 'M', u'ί'), + (0x1F78, 'V'), + (0x1F79, 'M', u'ό'), + (0x1F7A, 'V'), + (0x1F7B, 'M', u'ύ'), + (0x1F7C, 'V'), + (0x1F7D, 'M', u'ώ'), + (0x1F7E, 'X'), + (0x1F80, 'M', u'ἀι'), + (0x1F81, 'M', u'ἁι'), + (0x1F82, 'M', u'ἂι'), + (0x1F83, 'M', u'ἃι'), + (0x1F84, 'M', u'ἄι'), + ] + +def _seg_20(): + return [ + (0x1F85, 'M', u'ἅι'), + (0x1F86, 'M', u'ἆι'), + (0x1F87, 'M', u'ἇι'), + (0x1F88, 'M', u'ἀι'), + (0x1F89, 'M', u'ἁι'), + (0x1F8A, 'M', u'ἂι'), + (0x1F8B, 'M', u'ἃι'), + (0x1F8C, 'M', u'ἄι'), + (0x1F8D, 'M', u'ἅι'), + (0x1F8E, 'M', u'ἆι'), + (0x1F8F, 'M', u'ἇι'), + (0x1F90, 'M', u'ἠι'), + (0x1F91, 'M', u'ἡι'), + (0x1F92, 'M', u'ἢι'), + (0x1F93, 'M', u'ἣι'), + (0x1F94, 'M', u'ἤι'), + (0x1F95, 'M', u'ἥι'), + (0x1F96, 'M', u'ἦι'), + (0x1F97, 'M', u'ἧι'), + (0x1F98, 'M', u'ἠι'), + (0x1F99, 'M', u'ἡι'), + (0x1F9A, 'M', u'ἢι'), + (0x1F9B, 'M', u'ἣι'), + (0x1F9C, 'M', u'ἤι'), + (0x1F9D, 'M', u'ἥι'), + (0x1F9E, 'M', u'ἦι'), + (0x1F9F, 'M', u'ἧι'), + (0x1FA0, 'M', u'ὠι'), + (0x1FA1, 'M', u'ὡι'), + (0x1FA2, 'M', u'ὢι'), + (0x1FA3, 'M', u'ὣι'), + (0x1FA4, 'M', u'ὤι'), + (0x1FA5, 'M', u'ὥι'), + (0x1FA6, 'M', u'ὦι'), + (0x1FA7, 'M', u'ὧι'), + (0x1FA8, 'M', u'ὠι'), + (0x1FA9, 'M', u'ὡι'), + (0x1FAA, 'M', u'ὢι'), + (0x1FAB, 'M', u'ὣι'), + (0x1FAC, 'M', u'ὤι'), + (0x1FAD, 'M', u'ὥι'), + (0x1FAE, 'M', u'ὦι'), + (0x1FAF, 'M', u'ὧι'), + (0x1FB0, 'V'), + (0x1FB2, 'M', u'ὰι'), + (0x1FB3, 'M', u'αι'), + (0x1FB4, 'M', u'άι'), + (0x1FB5, 'X'), + (0x1FB6, 'V'), + (0x1FB7, 'M', u'ᾶι'), + (0x1FB8, 'M', u'ᾰ'), + (0x1FB9, 'M', u'ᾱ'), + (0x1FBA, 'M', u'ὰ'), + (0x1FBB, 'M', u'ά'), + (0x1FBC, 'M', u'αι'), + (0x1FBD, '3', u' ̓'), + (0x1FBE, 'M', u'ι'), + (0x1FBF, '3', u' ̓'), + (0x1FC0, '3', u' ͂'), + (0x1FC1, '3', u' ̈͂'), + (0x1FC2, 'M', u'ὴι'), + (0x1FC3, 'M', u'ηι'), + (0x1FC4, 'M', u'ήι'), + (0x1FC5, 'X'), + (0x1FC6, 'V'), + (0x1FC7, 'M', u'ῆι'), + (0x1FC8, 'M', u'ὲ'), + (0x1FC9, 'M', u'έ'), + (0x1FCA, 'M', u'ὴ'), + (0x1FCB, 'M', u'ή'), + (0x1FCC, 'M', u'ηι'), + (0x1FCD, '3', u' ̓̀'), + (0x1FCE, '3', u' ̓́'), + (0x1FCF, '3', u' ̓͂'), + (0x1FD0, 'V'), + (0x1FD3, 'M', u'ΐ'), + (0x1FD4, 'X'), + (0x1FD6, 'V'), + (0x1FD8, 'M', u'ῐ'), + (0x1FD9, 'M', u'ῑ'), + (0x1FDA, 'M', u'ὶ'), + (0x1FDB, 'M', u'ί'), + (0x1FDC, 'X'), + (0x1FDD, '3', u' ̔̀'), + (0x1FDE, '3', u' ̔́'), + (0x1FDF, '3', u' ̔͂'), + (0x1FE0, 'V'), + (0x1FE3, 'M', u'ΰ'), + (0x1FE4, 'V'), + (0x1FE8, 'M', u'ῠ'), + (0x1FE9, 'M', u'ῡ'), + (0x1FEA, 'M', u'ὺ'), + (0x1FEB, 'M', u'ύ'), + (0x1FEC, 'M', u'ῥ'), + (0x1FED, '3', u' ̈̀'), + (0x1FEE, '3', u' ̈́'), + (0x1FEF, '3', u'`'), + (0x1FF0, 'X'), + (0x1FF2, 'M', u'ὼι'), + (0x1FF3, 'M', u'ωι'), + ] + +def _seg_21(): + return [ + (0x1FF4, 'M', u'ώι'), + (0x1FF5, 'X'), + (0x1FF6, 'V'), + (0x1FF7, 'M', u'ῶι'), + (0x1FF8, 'M', u'ὸ'), + (0x1FF9, 'M', u'ό'), + (0x1FFA, 'M', u'ὼ'), + (0x1FFB, 'M', u'ώ'), + (0x1FFC, 'M', u'ωι'), + (0x1FFD, '3', u' ́'), + (0x1FFE, '3', u' ̔'), + (0x1FFF, 'X'), + (0x2000, '3', u' '), + (0x200B, 'I'), + (0x200C, 'D', u''), + (0x200E, 'X'), + (0x2010, 'V'), + (0x2011, 'M', u'‐'), + (0x2012, 'V'), + (0x2017, '3', u' ̳'), + (0x2018, 'V'), + (0x2024, 'X'), + (0x2027, 'V'), + (0x2028, 'X'), + (0x202F, '3', u' '), + (0x2030, 'V'), + (0x2033, 'M', u'′′'), + (0x2034, 'M', u'′′′'), + (0x2035, 'V'), + (0x2036, 'M', u'‵‵'), + (0x2037, 'M', u'‵‵‵'), + (0x2038, 'V'), + (0x203C, '3', u'!!'), + (0x203D, 'V'), + (0x203E, '3', u' ̅'), + (0x203F, 'V'), + (0x2047, '3', u'??'), + (0x2048, '3', u'?!'), + (0x2049, '3', u'!?'), + (0x204A, 'V'), + (0x2057, 'M', u'′′′′'), + (0x2058, 'V'), + (0x205F, '3', u' '), + (0x2060, 'I'), + (0x2061, 'X'), + (0x2064, 'I'), + (0x2065, 'X'), + (0x2070, 'M', u'0'), + (0x2071, 'M', u'i'), + (0x2072, 'X'), + (0x2074, 'M', u'4'), + (0x2075, 'M', u'5'), + (0x2076, 'M', u'6'), + (0x2077, 'M', u'7'), + (0x2078, 'M', u'8'), + (0x2079, 'M', u'9'), + (0x207A, '3', u'+'), + (0x207B, 'M', u'−'), + (0x207C, '3', u'='), + (0x207D, '3', u'('), + (0x207E, '3', u')'), + (0x207F, 'M', u'n'), + (0x2080, 'M', u'0'), + (0x2081, 'M', u'1'), + (0x2082, 'M', u'2'), + (0x2083, 'M', u'3'), + (0x2084, 'M', u'4'), + (0x2085, 'M', u'5'), + (0x2086, 'M', u'6'), + (0x2087, 'M', u'7'), + (0x2088, 'M', u'8'), + (0x2089, 'M', u'9'), + (0x208A, '3', u'+'), + (0x208B, 'M', u'−'), + (0x208C, '3', u'='), + (0x208D, '3', u'('), + (0x208E, '3', u')'), + (0x208F, 'X'), + (0x2090, 'M', u'a'), + (0x2091, 'M', u'e'), + (0x2092, 'M', u'o'), + (0x2093, 'M', u'x'), + (0x2094, 'M', u'ə'), + (0x2095, 'M', u'h'), + (0x2096, 'M', u'k'), + (0x2097, 'M', u'l'), + (0x2098, 'M', u'm'), + (0x2099, 'M', u'n'), + (0x209A, 'M', u'p'), + (0x209B, 'M', u's'), + (0x209C, 'M', u't'), + (0x209D, 'X'), + (0x20A0, 'V'), + (0x20A8, 'M', u'rs'), + (0x20A9, 'V'), + (0x20C0, 'X'), + (0x20D0, 'V'), + (0x20F1, 'X'), + (0x2100, '3', u'a/c'), + (0x2101, '3', u'a/s'), + ] + +def _seg_22(): + return [ + (0x2102, 'M', u'c'), + (0x2103, 'M', u'°c'), + (0x2104, 'V'), + (0x2105, '3', u'c/o'), + (0x2106, '3', u'c/u'), + (0x2107, 'M', u'ɛ'), + (0x2108, 'V'), + (0x2109, 'M', u'°f'), + (0x210A, 'M', u'g'), + (0x210B, 'M', u'h'), + (0x210F, 'M', u'ħ'), + (0x2110, 'M', u'i'), + (0x2112, 'M', u'l'), + (0x2114, 'V'), + (0x2115, 'M', u'n'), + (0x2116, 'M', u'no'), + (0x2117, 'V'), + (0x2119, 'M', u'p'), + (0x211A, 'M', u'q'), + (0x211B, 'M', u'r'), + (0x211E, 'V'), + (0x2120, 'M', u'sm'), + (0x2121, 'M', u'tel'), + (0x2122, 'M', u'tm'), + (0x2123, 'V'), + (0x2124, 'M', u'z'), + (0x2125, 'V'), + (0x2126, 'M', u'ω'), + (0x2127, 'V'), + (0x2128, 'M', u'z'), + (0x2129, 'V'), + (0x212A, 'M', u'k'), + (0x212B, 'M', u'å'), + (0x212C, 'M', u'b'), + (0x212D, 'M', u'c'), + (0x212E, 'V'), + (0x212F, 'M', u'e'), + (0x2131, 'M', u'f'), + (0x2132, 'X'), + (0x2133, 'M', u'm'), + (0x2134, 'M', u'o'), + (0x2135, 'M', u'א'), + (0x2136, 'M', u'ב'), + (0x2137, 'M', u'ג'), + (0x2138, 'M', u'ד'), + (0x2139, 'M', u'i'), + (0x213A, 'V'), + (0x213B, 'M', u'fax'), + (0x213C, 'M', u'π'), + (0x213D, 'M', u'γ'), + (0x213F, 'M', u'π'), + (0x2140, 'M', u'∑'), + (0x2141, 'V'), + (0x2145, 'M', u'd'), + (0x2147, 'M', u'e'), + (0x2148, 'M', u'i'), + (0x2149, 'M', u'j'), + (0x214A, 'V'), + (0x2150, 'M', u'1⁄7'), + (0x2151, 'M', u'1⁄9'), + (0x2152, 'M', u'1⁄10'), + (0x2153, 'M', u'1⁄3'), + (0x2154, 'M', u'2⁄3'), + (0x2155, 'M', u'1⁄5'), + (0x2156, 'M', u'2⁄5'), + (0x2157, 'M', u'3⁄5'), + (0x2158, 'M', u'4⁄5'), + (0x2159, 'M', u'1⁄6'), + (0x215A, 'M', u'5⁄6'), + (0x215B, 'M', u'1⁄8'), + (0x215C, 'M', u'3⁄8'), + (0x215D, 'M', u'5⁄8'), + (0x215E, 'M', u'7⁄8'), + (0x215F, 'M', u'1⁄'), + (0x2160, 'M', u'i'), + (0x2161, 'M', u'ii'), + (0x2162, 'M', u'iii'), + (0x2163, 'M', u'iv'), + (0x2164, 'M', u'v'), + (0x2165, 'M', u'vi'), + (0x2166, 'M', u'vii'), + (0x2167, 'M', u'viii'), + (0x2168, 'M', u'ix'), + (0x2169, 'M', u'x'), + (0x216A, 'M', u'xi'), + (0x216B, 'M', u'xii'), + (0x216C, 'M', u'l'), + (0x216D, 'M', u'c'), + (0x216E, 'M', u'd'), + (0x216F, 'M', u'm'), + (0x2170, 'M', u'i'), + (0x2171, 'M', u'ii'), + (0x2172, 'M', u'iii'), + (0x2173, 'M', u'iv'), + (0x2174, 'M', u'v'), + (0x2175, 'M', u'vi'), + (0x2176, 'M', u'vii'), + (0x2177, 'M', u'viii'), + (0x2178, 'M', u'ix'), + (0x2179, 'M', u'x'), + ] + +def _seg_23(): + return [ + (0x217A, 'M', u'xi'), + (0x217B, 'M', u'xii'), + (0x217C, 'M', u'l'), + (0x217D, 'M', u'c'), + (0x217E, 'M', u'd'), + (0x217F, 'M', u'm'), + (0x2180, 'V'), + (0x2183, 'X'), + (0x2184, 'V'), + (0x2189, 'M', u'0⁄3'), + (0x218A, 'V'), + (0x218C, 'X'), + (0x2190, 'V'), + (0x222C, 'M', u'∫∫'), + (0x222D, 'M', u'∫∫∫'), + (0x222E, 'V'), + (0x222F, 'M', u'∮∮'), + (0x2230, 'M', u'∮∮∮'), + (0x2231, 'V'), + (0x2260, '3'), + (0x2261, 'V'), + (0x226E, '3'), + (0x2270, 'V'), + (0x2329, 'M', u'〈'), + (0x232A, 'M', u'〉'), + (0x232B, 'V'), + (0x2427, 'X'), + (0x2440, 'V'), + (0x244B, 'X'), + (0x2460, 'M', u'1'), + (0x2461, 'M', u'2'), + (0x2462, 'M', u'3'), + (0x2463, 'M', u'4'), + (0x2464, 'M', u'5'), + (0x2465, 'M', u'6'), + (0x2466, 'M', u'7'), + (0x2467, 'M', u'8'), + (0x2468, 'M', u'9'), + (0x2469, 'M', u'10'), + (0x246A, 'M', u'11'), + (0x246B, 'M', u'12'), + (0x246C, 'M', u'13'), + (0x246D, 'M', u'14'), + (0x246E, 'M', u'15'), + (0x246F, 'M', u'16'), + (0x2470, 'M', u'17'), + (0x2471, 'M', u'18'), + (0x2472, 'M', u'19'), + (0x2473, 'M', u'20'), + (0x2474, '3', u'(1)'), + (0x2475, '3', u'(2)'), + (0x2476, '3', u'(3)'), + (0x2477, '3', u'(4)'), + (0x2478, '3', u'(5)'), + (0x2479, '3', u'(6)'), + (0x247A, '3', u'(7)'), + (0x247B, '3', u'(8)'), + (0x247C, '3', u'(9)'), + (0x247D, '3', u'(10)'), + (0x247E, '3', u'(11)'), + (0x247F, '3', u'(12)'), + (0x2480, '3', u'(13)'), + (0x2481, '3', u'(14)'), + (0x2482, '3', u'(15)'), + (0x2483, '3', u'(16)'), + (0x2484, '3', u'(17)'), + (0x2485, '3', u'(18)'), + (0x2486, '3', u'(19)'), + (0x2487, '3', u'(20)'), + (0x2488, 'X'), + (0x249C, '3', u'(a)'), + (0x249D, '3', u'(b)'), + (0x249E, '3', u'(c)'), + (0x249F, '3', u'(d)'), + (0x24A0, '3', u'(e)'), + (0x24A1, '3', u'(f)'), + (0x24A2, '3', u'(g)'), + (0x24A3, '3', u'(h)'), + (0x24A4, '3', u'(i)'), + (0x24A5, '3', u'(j)'), + (0x24A6, '3', u'(k)'), + (0x24A7, '3', u'(l)'), + (0x24A8, '3', u'(m)'), + (0x24A9, '3', u'(n)'), + (0x24AA, '3', u'(o)'), + (0x24AB, '3', u'(p)'), + (0x24AC, '3', u'(q)'), + (0x24AD, '3', u'(r)'), + (0x24AE, '3', u'(s)'), + (0x24AF, '3', u'(t)'), + (0x24B0, '3', u'(u)'), + (0x24B1, '3', u'(v)'), + (0x24B2, '3', u'(w)'), + (0x24B3, '3', u'(x)'), + (0x24B4, '3', u'(y)'), + (0x24B5, '3', u'(z)'), + (0x24B6, 'M', u'a'), + (0x24B7, 'M', u'b'), + (0x24B8, 'M', u'c'), + (0x24B9, 'M', u'd'), + ] + +def _seg_24(): + return [ + (0x24BA, 'M', u'e'), + (0x24BB, 'M', u'f'), + (0x24BC, 'M', u'g'), + (0x24BD, 'M', u'h'), + (0x24BE, 'M', u'i'), + (0x24BF, 'M', u'j'), + (0x24C0, 'M', u'k'), + (0x24C1, 'M', u'l'), + (0x24C2, 'M', u'm'), + (0x24C3, 'M', u'n'), + (0x24C4, 'M', u'o'), + (0x24C5, 'M', u'p'), + (0x24C6, 'M', u'q'), + (0x24C7, 'M', u'r'), + (0x24C8, 'M', u's'), + (0x24C9, 'M', u't'), + (0x24CA, 'M', u'u'), + (0x24CB, 'M', u'v'), + (0x24CC, 'M', u'w'), + (0x24CD, 'M', u'x'), + (0x24CE, 'M', u'y'), + (0x24CF, 'M', u'z'), + (0x24D0, 'M', u'a'), + (0x24D1, 'M', u'b'), + (0x24D2, 'M', u'c'), + (0x24D3, 'M', u'd'), + (0x24D4, 'M', u'e'), + (0x24D5, 'M', u'f'), + (0x24D6, 'M', u'g'), + (0x24D7, 'M', u'h'), + (0x24D8, 'M', u'i'), + (0x24D9, 'M', u'j'), + (0x24DA, 'M', u'k'), + (0x24DB, 'M', u'l'), + (0x24DC, 'M', u'm'), + (0x24DD, 'M', u'n'), + (0x24DE, 'M', u'o'), + (0x24DF, 'M', u'p'), + (0x24E0, 'M', u'q'), + (0x24E1, 'M', u'r'), + (0x24E2, 'M', u's'), + (0x24E3, 'M', u't'), + (0x24E4, 'M', u'u'), + (0x24E5, 'M', u'v'), + (0x24E6, 'M', u'w'), + (0x24E7, 'M', u'x'), + (0x24E8, 'M', u'y'), + (0x24E9, 'M', u'z'), + (0x24EA, 'M', u'0'), + (0x24EB, 'V'), + (0x2A0C, 'M', u'∫∫∫∫'), + (0x2A0D, 'V'), + (0x2A74, '3', u'::='), + (0x2A75, '3', u'=='), + (0x2A76, '3', u'==='), + (0x2A77, 'V'), + (0x2ADC, 'M', u'⫝̸'), + (0x2ADD, 'V'), + (0x2B74, 'X'), + (0x2B76, 'V'), + (0x2B96, 'X'), + (0x2B97, 'V'), + (0x2C00, 'M', u'ⰰ'), + (0x2C01, 'M', u'ⰱ'), + (0x2C02, 'M', u'ⰲ'), + (0x2C03, 'M', u'ⰳ'), + (0x2C04, 'M', u'ⰴ'), + (0x2C05, 'M', u'ⰵ'), + (0x2C06, 'M', u'ⰶ'), + (0x2C07, 'M', u'ⰷ'), + (0x2C08, 'M', u'ⰸ'), + (0x2C09, 'M', u'ⰹ'), + (0x2C0A, 'M', u'ⰺ'), + (0x2C0B, 'M', u'ⰻ'), + (0x2C0C, 'M', u'ⰼ'), + (0x2C0D, 'M', u'ⰽ'), + (0x2C0E, 'M', u'ⰾ'), + (0x2C0F, 'M', u'ⰿ'), + (0x2C10, 'M', u'ⱀ'), + (0x2C11, 'M', u'ⱁ'), + (0x2C12, 'M', u'ⱂ'), + (0x2C13, 'M', u'ⱃ'), + (0x2C14, 'M', u'ⱄ'), + (0x2C15, 'M', u'ⱅ'), + (0x2C16, 'M', u'ⱆ'), + (0x2C17, 'M', u'ⱇ'), + (0x2C18, 'M', u'ⱈ'), + (0x2C19, 'M', u'ⱉ'), + (0x2C1A, 'M', u'ⱊ'), + (0x2C1B, 'M', u'ⱋ'), + (0x2C1C, 'M', u'ⱌ'), + (0x2C1D, 'M', u'ⱍ'), + (0x2C1E, 'M', u'ⱎ'), + (0x2C1F, 'M', u'ⱏ'), + (0x2C20, 'M', u'ⱐ'), + (0x2C21, 'M', u'ⱑ'), + (0x2C22, 'M', u'ⱒ'), + (0x2C23, 'M', u'ⱓ'), + (0x2C24, 'M', u'ⱔ'), + (0x2C25, 'M', u'ⱕ'), + ] + +def _seg_25(): + return [ + (0x2C26, 'M', u'ⱖ'), + (0x2C27, 'M', u'ⱗ'), + (0x2C28, 'M', u'ⱘ'), + (0x2C29, 'M', u'ⱙ'), + (0x2C2A, 'M', u'ⱚ'), + (0x2C2B, 'M', u'ⱛ'), + (0x2C2C, 'M', u'ⱜ'), + (0x2C2D, 'M', u'ⱝ'), + (0x2C2E, 'M', u'ⱞ'), + (0x2C2F, 'X'), + (0x2C30, 'V'), + (0x2C5F, 'X'), + (0x2C60, 'M', u'ⱡ'), + (0x2C61, 'V'), + (0x2C62, 'M', u'ɫ'), + (0x2C63, 'M', u'ᵽ'), + (0x2C64, 'M', u'ɽ'), + (0x2C65, 'V'), + (0x2C67, 'M', u'ⱨ'), + (0x2C68, 'V'), + (0x2C69, 'M', u'ⱪ'), + (0x2C6A, 'V'), + (0x2C6B, 'M', u'ⱬ'), + (0x2C6C, 'V'), + (0x2C6D, 'M', u'ɑ'), + (0x2C6E, 'M', u'ɱ'), + (0x2C6F, 'M', u'ɐ'), + (0x2C70, 'M', u'ɒ'), + (0x2C71, 'V'), + (0x2C72, 'M', u'ⱳ'), + (0x2C73, 'V'), + (0x2C75, 'M', u'ⱶ'), + (0x2C76, 'V'), + (0x2C7C, 'M', u'j'), + (0x2C7D, 'M', u'v'), + (0x2C7E, 'M', u'ȿ'), + (0x2C7F, 'M', u'ɀ'), + (0x2C80, 'M', u'ⲁ'), + (0x2C81, 'V'), + (0x2C82, 'M', u'ⲃ'), + (0x2C83, 'V'), + (0x2C84, 'M', u'ⲅ'), + (0x2C85, 'V'), + (0x2C86, 'M', u'ⲇ'), + (0x2C87, 'V'), + (0x2C88, 'M', u'ⲉ'), + (0x2C89, 'V'), + (0x2C8A, 'M', u'ⲋ'), + (0x2C8B, 'V'), + (0x2C8C, 'M', u'ⲍ'), + (0x2C8D, 'V'), + (0x2C8E, 'M', u'ⲏ'), + (0x2C8F, 'V'), + (0x2C90, 'M', u'ⲑ'), + (0x2C91, 'V'), + (0x2C92, 'M', u'ⲓ'), + (0x2C93, 'V'), + (0x2C94, 'M', u'ⲕ'), + (0x2C95, 'V'), + (0x2C96, 'M', u'ⲗ'), + (0x2C97, 'V'), + (0x2C98, 'M', u'ⲙ'), + (0x2C99, 'V'), + (0x2C9A, 'M', u'ⲛ'), + (0x2C9B, 'V'), + (0x2C9C, 'M', u'ⲝ'), + (0x2C9D, 'V'), + (0x2C9E, 'M', u'ⲟ'), + (0x2C9F, 'V'), + (0x2CA0, 'M', u'ⲡ'), + (0x2CA1, 'V'), + (0x2CA2, 'M', u'ⲣ'), + (0x2CA3, 'V'), + (0x2CA4, 'M', u'ⲥ'), + (0x2CA5, 'V'), + (0x2CA6, 'M', u'ⲧ'), + (0x2CA7, 'V'), + (0x2CA8, 'M', u'ⲩ'), + (0x2CA9, 'V'), + (0x2CAA, 'M', u'ⲫ'), + (0x2CAB, 'V'), + (0x2CAC, 'M', u'ⲭ'), + (0x2CAD, 'V'), + (0x2CAE, 'M', u'ⲯ'), + (0x2CAF, 'V'), + (0x2CB0, 'M', u'ⲱ'), + (0x2CB1, 'V'), + (0x2CB2, 'M', u'ⲳ'), + (0x2CB3, 'V'), + (0x2CB4, 'M', u'ⲵ'), + (0x2CB5, 'V'), + (0x2CB6, 'M', u'ⲷ'), + (0x2CB7, 'V'), + (0x2CB8, 'M', u'ⲹ'), + (0x2CB9, 'V'), + (0x2CBA, 'M', u'ⲻ'), + (0x2CBB, 'V'), + (0x2CBC, 'M', u'ⲽ'), + (0x2CBD, 'V'), + (0x2CBE, 'M', u'ⲿ'), + ] + +def _seg_26(): + return [ + (0x2CBF, 'V'), + (0x2CC0, 'M', u'ⳁ'), + (0x2CC1, 'V'), + (0x2CC2, 'M', u'ⳃ'), + (0x2CC3, 'V'), + (0x2CC4, 'M', u'ⳅ'), + (0x2CC5, 'V'), + (0x2CC6, 'M', u'ⳇ'), + (0x2CC7, 'V'), + (0x2CC8, 'M', u'ⳉ'), + (0x2CC9, 'V'), + (0x2CCA, 'M', u'ⳋ'), + (0x2CCB, 'V'), + (0x2CCC, 'M', u'ⳍ'), + (0x2CCD, 'V'), + (0x2CCE, 'M', u'ⳏ'), + (0x2CCF, 'V'), + (0x2CD0, 'M', u'ⳑ'), + (0x2CD1, 'V'), + (0x2CD2, 'M', u'ⳓ'), + (0x2CD3, 'V'), + (0x2CD4, 'M', u'ⳕ'), + (0x2CD5, 'V'), + (0x2CD6, 'M', u'ⳗ'), + (0x2CD7, 'V'), + (0x2CD8, 'M', u'ⳙ'), + (0x2CD9, 'V'), + (0x2CDA, 'M', u'ⳛ'), + (0x2CDB, 'V'), + (0x2CDC, 'M', u'ⳝ'), + (0x2CDD, 'V'), + (0x2CDE, 'M', u'ⳟ'), + (0x2CDF, 'V'), + (0x2CE0, 'M', u'ⳡ'), + (0x2CE1, 'V'), + (0x2CE2, 'M', u'ⳣ'), + (0x2CE3, 'V'), + (0x2CEB, 'M', u'ⳬ'), + (0x2CEC, 'V'), + (0x2CED, 'M', u'ⳮ'), + (0x2CEE, 'V'), + (0x2CF2, 'M', u'ⳳ'), + (0x2CF3, 'V'), + (0x2CF4, 'X'), + (0x2CF9, 'V'), + (0x2D26, 'X'), + (0x2D27, 'V'), + (0x2D28, 'X'), + (0x2D2D, 'V'), + (0x2D2E, 'X'), + (0x2D30, 'V'), + (0x2D68, 'X'), + (0x2D6F, 'M', u'ⵡ'), + (0x2D70, 'V'), + (0x2D71, 'X'), + (0x2D7F, 'V'), + (0x2D97, 'X'), + (0x2DA0, 'V'), + (0x2DA7, 'X'), + (0x2DA8, 'V'), + (0x2DAF, 'X'), + (0x2DB0, 'V'), + (0x2DB7, 'X'), + (0x2DB8, 'V'), + (0x2DBF, 'X'), + (0x2DC0, 'V'), + (0x2DC7, 'X'), + (0x2DC8, 'V'), + (0x2DCF, 'X'), + (0x2DD0, 'V'), + (0x2DD7, 'X'), + (0x2DD8, 'V'), + (0x2DDF, 'X'), + (0x2DE0, 'V'), + (0x2E53, 'X'), + (0x2E80, 'V'), + (0x2E9A, 'X'), + (0x2E9B, 'V'), + (0x2E9F, 'M', u'母'), + (0x2EA0, 'V'), + (0x2EF3, 'M', u'龟'), + (0x2EF4, 'X'), + (0x2F00, 'M', u'一'), + (0x2F01, 'M', u'丨'), + (0x2F02, 'M', u'丶'), + (0x2F03, 'M', u'丿'), + (0x2F04, 'M', u'乙'), + (0x2F05, 'M', u'亅'), + (0x2F06, 'M', u'二'), + (0x2F07, 'M', u'亠'), + (0x2F08, 'M', u'人'), + (0x2F09, 'M', u'儿'), + (0x2F0A, 'M', u'入'), + (0x2F0B, 'M', u'八'), + (0x2F0C, 'M', u'冂'), + (0x2F0D, 'M', u'冖'), + (0x2F0E, 'M', u'冫'), + (0x2F0F, 'M', u'几'), + (0x2F10, 'M', u'凵'), + (0x2F11, 'M', u'刀'), + ] + +def _seg_27(): + return [ + (0x2F12, 'M', u'力'), + (0x2F13, 'M', u'勹'), + (0x2F14, 'M', u'匕'), + (0x2F15, 'M', u'匚'), + (0x2F16, 'M', u'匸'), + (0x2F17, 'M', u'十'), + (0x2F18, 'M', u'卜'), + (0x2F19, 'M', u'卩'), + (0x2F1A, 'M', u'厂'), + (0x2F1B, 'M', u'厶'), + (0x2F1C, 'M', u'又'), + (0x2F1D, 'M', u'口'), + (0x2F1E, 'M', u'囗'), + (0x2F1F, 'M', u'土'), + (0x2F20, 'M', u'士'), + (0x2F21, 'M', u'夂'), + (0x2F22, 'M', u'夊'), + (0x2F23, 'M', u'夕'), + (0x2F24, 'M', u'大'), + (0x2F25, 'M', u'女'), + (0x2F26, 'M', u'子'), + (0x2F27, 'M', u'宀'), + (0x2F28, 'M', u'寸'), + (0x2F29, 'M', u'小'), + (0x2F2A, 'M', u'尢'), + (0x2F2B, 'M', u'尸'), + (0x2F2C, 'M', u'屮'), + (0x2F2D, 'M', u'山'), + (0x2F2E, 'M', u'巛'), + (0x2F2F, 'M', u'工'), + (0x2F30, 'M', u'己'), + (0x2F31, 'M', u'巾'), + (0x2F32, 'M', u'干'), + (0x2F33, 'M', u'幺'), + (0x2F34, 'M', u'广'), + (0x2F35, 'M', u'廴'), + (0x2F36, 'M', u'廾'), + (0x2F37, 'M', u'弋'), + (0x2F38, 'M', u'弓'), + (0x2F39, 'M', u'彐'), + (0x2F3A, 'M', u'彡'), + (0x2F3B, 'M', u'彳'), + (0x2F3C, 'M', u'心'), + (0x2F3D, 'M', u'戈'), + (0x2F3E, 'M', u'戶'), + (0x2F3F, 'M', u'手'), + (0x2F40, 'M', u'支'), + (0x2F41, 'M', u'攴'), + (0x2F42, 'M', u'文'), + (0x2F43, 'M', u'斗'), + (0x2F44, 'M', u'斤'), + (0x2F45, 'M', u'方'), + (0x2F46, 'M', u'无'), + (0x2F47, 'M', u'日'), + (0x2F48, 'M', u'曰'), + (0x2F49, 'M', u'月'), + (0x2F4A, 'M', u'木'), + (0x2F4B, 'M', u'欠'), + (0x2F4C, 'M', u'止'), + (0x2F4D, 'M', u'歹'), + (0x2F4E, 'M', u'殳'), + (0x2F4F, 'M', u'毋'), + (0x2F50, 'M', u'比'), + (0x2F51, 'M', u'毛'), + (0x2F52, 'M', u'氏'), + (0x2F53, 'M', u'气'), + (0x2F54, 'M', u'水'), + (0x2F55, 'M', u'火'), + (0x2F56, 'M', u'爪'), + (0x2F57, 'M', u'父'), + (0x2F58, 'M', u'爻'), + (0x2F59, 'M', u'爿'), + (0x2F5A, 'M', u'片'), + (0x2F5B, 'M', u'牙'), + (0x2F5C, 'M', u'牛'), + (0x2F5D, 'M', u'犬'), + (0x2F5E, 'M', u'玄'), + (0x2F5F, 'M', u'玉'), + (0x2F60, 'M', u'瓜'), + (0x2F61, 'M', u'瓦'), + (0x2F62, 'M', u'甘'), + (0x2F63, 'M', u'生'), + (0x2F64, 'M', u'用'), + (0x2F65, 'M', u'田'), + (0x2F66, 'M', u'疋'), + (0x2F67, 'M', u'疒'), + (0x2F68, 'M', u'癶'), + (0x2F69, 'M', u'白'), + (0x2F6A, 'M', u'皮'), + (0x2F6B, 'M', u'皿'), + (0x2F6C, 'M', u'目'), + (0x2F6D, 'M', u'矛'), + (0x2F6E, 'M', u'矢'), + (0x2F6F, 'M', u'石'), + (0x2F70, 'M', u'示'), + (0x2F71, 'M', u'禸'), + (0x2F72, 'M', u'禾'), + (0x2F73, 'M', u'穴'), + (0x2F74, 'M', u'立'), + (0x2F75, 'M', u'竹'), + ] + +def _seg_28(): + return [ + (0x2F76, 'M', u'米'), + (0x2F77, 'M', u'糸'), + (0x2F78, 'M', u'缶'), + (0x2F79, 'M', u'网'), + (0x2F7A, 'M', u'羊'), + (0x2F7B, 'M', u'羽'), + (0x2F7C, 'M', u'老'), + (0x2F7D, 'M', u'而'), + (0x2F7E, 'M', u'耒'), + (0x2F7F, 'M', u'耳'), + (0x2F80, 'M', u'聿'), + (0x2F81, 'M', u'肉'), + (0x2F82, 'M', u'臣'), + (0x2F83, 'M', u'自'), + (0x2F84, 'M', u'至'), + (0x2F85, 'M', u'臼'), + (0x2F86, 'M', u'舌'), + (0x2F87, 'M', u'舛'), + (0x2F88, 'M', u'舟'), + (0x2F89, 'M', u'艮'), + (0x2F8A, 'M', u'色'), + (0x2F8B, 'M', u'艸'), + (0x2F8C, 'M', u'虍'), + (0x2F8D, 'M', u'虫'), + (0x2F8E, 'M', u'血'), + (0x2F8F, 'M', u'行'), + (0x2F90, 'M', u'衣'), + (0x2F91, 'M', u'襾'), + (0x2F92, 'M', u'見'), + (0x2F93, 'M', u'角'), + (0x2F94, 'M', u'言'), + (0x2F95, 'M', u'谷'), + (0x2F96, 'M', u'豆'), + (0x2F97, 'M', u'豕'), + (0x2F98, 'M', u'豸'), + (0x2F99, 'M', u'貝'), + (0x2F9A, 'M', u'赤'), + (0x2F9B, 'M', u'走'), + (0x2F9C, 'M', u'足'), + (0x2F9D, 'M', u'身'), + (0x2F9E, 'M', u'車'), + (0x2F9F, 'M', u'辛'), + (0x2FA0, 'M', u'辰'), + (0x2FA1, 'M', u'辵'), + (0x2FA2, 'M', u'邑'), + (0x2FA3, 'M', u'酉'), + (0x2FA4, 'M', u'釆'), + (0x2FA5, 'M', u'里'), + (0x2FA6, 'M', u'金'), + (0x2FA7, 'M', u'長'), + (0x2FA8, 'M', u'門'), + (0x2FA9, 'M', u'阜'), + (0x2FAA, 'M', u'隶'), + (0x2FAB, 'M', u'隹'), + (0x2FAC, 'M', u'雨'), + (0x2FAD, 'M', u'靑'), + (0x2FAE, 'M', u'非'), + (0x2FAF, 'M', u'面'), + (0x2FB0, 'M', u'革'), + (0x2FB1, 'M', u'韋'), + (0x2FB2, 'M', u'韭'), + (0x2FB3, 'M', u'音'), + (0x2FB4, 'M', u'頁'), + (0x2FB5, 'M', u'風'), + (0x2FB6, 'M', u'飛'), + (0x2FB7, 'M', u'食'), + (0x2FB8, 'M', u'首'), + (0x2FB9, 'M', u'香'), + (0x2FBA, 'M', u'馬'), + (0x2FBB, 'M', u'骨'), + (0x2FBC, 'M', u'高'), + (0x2FBD, 'M', u'髟'), + (0x2FBE, 'M', u'鬥'), + (0x2FBF, 'M', u'鬯'), + (0x2FC0, 'M', u'鬲'), + (0x2FC1, 'M', u'鬼'), + (0x2FC2, 'M', u'魚'), + (0x2FC3, 'M', u'鳥'), + (0x2FC4, 'M', u'鹵'), + (0x2FC5, 'M', u'鹿'), + (0x2FC6, 'M', u'麥'), + (0x2FC7, 'M', u'麻'), + (0x2FC8, 'M', u'黃'), + (0x2FC9, 'M', u'黍'), + (0x2FCA, 'M', u'黑'), + (0x2FCB, 'M', u'黹'), + (0x2FCC, 'M', u'黽'), + (0x2FCD, 'M', u'鼎'), + (0x2FCE, 'M', u'鼓'), + (0x2FCF, 'M', u'鼠'), + (0x2FD0, 'M', u'鼻'), + (0x2FD1, 'M', u'齊'), + (0x2FD2, 'M', u'齒'), + (0x2FD3, 'M', u'龍'), + (0x2FD4, 'M', u'龜'), + (0x2FD5, 'M', u'龠'), + (0x2FD6, 'X'), + (0x3000, '3', u' '), + (0x3001, 'V'), + (0x3002, 'M', u'.'), + ] + +def _seg_29(): + return [ + (0x3003, 'V'), + (0x3036, 'M', u'〒'), + (0x3037, 'V'), + (0x3038, 'M', u'十'), + (0x3039, 'M', u'卄'), + (0x303A, 'M', u'卅'), + (0x303B, 'V'), + (0x3040, 'X'), + (0x3041, 'V'), + (0x3097, 'X'), + (0x3099, 'V'), + (0x309B, '3', u' ゙'), + (0x309C, '3', u' ゚'), + (0x309D, 'V'), + (0x309F, 'M', u'より'), + (0x30A0, 'V'), + (0x30FF, 'M', u'コト'), + (0x3100, 'X'), + (0x3105, 'V'), + (0x3130, 'X'), + (0x3131, 'M', u'ᄀ'), + (0x3132, 'M', u'ᄁ'), + (0x3133, 'M', u'ᆪ'), + (0x3134, 'M', u'ᄂ'), + (0x3135, 'M', u'ᆬ'), + (0x3136, 'M', u'ᆭ'), + (0x3137, 'M', u'ᄃ'), + (0x3138, 'M', u'ᄄ'), + (0x3139, 'M', u'ᄅ'), + (0x313A, 'M', u'ᆰ'), + (0x313B, 'M', u'ᆱ'), + (0x313C, 'M', u'ᆲ'), + (0x313D, 'M', u'ᆳ'), + (0x313E, 'M', u'ᆴ'), + (0x313F, 'M', u'ᆵ'), + (0x3140, 'M', u'ᄚ'), + (0x3141, 'M', u'ᄆ'), + (0x3142, 'M', u'ᄇ'), + (0x3143, 'M', u'ᄈ'), + (0x3144, 'M', u'ᄡ'), + (0x3145, 'M', u'ᄉ'), + (0x3146, 'M', u'ᄊ'), + (0x3147, 'M', u'ᄋ'), + (0x3148, 'M', u'ᄌ'), + (0x3149, 'M', u'ᄍ'), + (0x314A, 'M', u'ᄎ'), + (0x314B, 'M', u'ᄏ'), + (0x314C, 'M', u'ᄐ'), + (0x314D, 'M', u'ᄑ'), + (0x314E, 'M', u'ᄒ'), + (0x314F, 'M', u'ᅡ'), + (0x3150, 'M', u'ᅢ'), + (0x3151, 'M', u'ᅣ'), + (0x3152, 'M', u'ᅤ'), + (0x3153, 'M', u'ᅥ'), + (0x3154, 'M', u'ᅦ'), + (0x3155, 'M', u'ᅧ'), + (0x3156, 'M', u'ᅨ'), + (0x3157, 'M', u'ᅩ'), + (0x3158, 'M', u'ᅪ'), + (0x3159, 'M', u'ᅫ'), + (0x315A, 'M', u'ᅬ'), + (0x315B, 'M', u'ᅭ'), + (0x315C, 'M', u'ᅮ'), + (0x315D, 'M', u'ᅯ'), + (0x315E, 'M', u'ᅰ'), + (0x315F, 'M', u'ᅱ'), + (0x3160, 'M', u'ᅲ'), + (0x3161, 'M', u'ᅳ'), + (0x3162, 'M', u'ᅴ'), + (0x3163, 'M', u'ᅵ'), + (0x3164, 'X'), + (0x3165, 'M', u'ᄔ'), + (0x3166, 'M', u'ᄕ'), + (0x3167, 'M', u'ᇇ'), + (0x3168, 'M', u'ᇈ'), + (0x3169, 'M', u'ᇌ'), + (0x316A, 'M', u'ᇎ'), + (0x316B, 'M', u'ᇓ'), + (0x316C, 'M', u'ᇗ'), + (0x316D, 'M', u'ᇙ'), + (0x316E, 'M', u'ᄜ'), + (0x316F, 'M', u'ᇝ'), + (0x3170, 'M', u'ᇟ'), + (0x3171, 'M', u'ᄝ'), + (0x3172, 'M', u'ᄞ'), + (0x3173, 'M', u'ᄠ'), + (0x3174, 'M', u'ᄢ'), + (0x3175, 'M', u'ᄣ'), + (0x3176, 'M', u'ᄧ'), + (0x3177, 'M', u'ᄩ'), + (0x3178, 'M', u'ᄫ'), + (0x3179, 'M', u'ᄬ'), + (0x317A, 'M', u'ᄭ'), + (0x317B, 'M', u'ᄮ'), + (0x317C, 'M', u'ᄯ'), + (0x317D, 'M', u'ᄲ'), + (0x317E, 'M', u'ᄶ'), + (0x317F, 'M', u'ᅀ'), + (0x3180, 'M', u'ᅇ'), + ] + +def _seg_30(): + return [ + (0x3181, 'M', u'ᅌ'), + (0x3182, 'M', u'ᇱ'), + (0x3183, 'M', u'ᇲ'), + (0x3184, 'M', u'ᅗ'), + (0x3185, 'M', u'ᅘ'), + (0x3186, 'M', u'ᅙ'), + (0x3187, 'M', u'ᆄ'), + (0x3188, 'M', u'ᆅ'), + (0x3189, 'M', u'ᆈ'), + (0x318A, 'M', u'ᆑ'), + (0x318B, 'M', u'ᆒ'), + (0x318C, 'M', u'ᆔ'), + (0x318D, 'M', u'ᆞ'), + (0x318E, 'M', u'ᆡ'), + (0x318F, 'X'), + (0x3190, 'V'), + (0x3192, 'M', u'一'), + (0x3193, 'M', u'二'), + (0x3194, 'M', u'三'), + (0x3195, 'M', u'四'), + (0x3196, 'M', u'上'), + (0x3197, 'M', u'中'), + (0x3198, 'M', u'下'), + (0x3199, 'M', u'甲'), + (0x319A, 'M', u'乙'), + (0x319B, 'M', u'丙'), + (0x319C, 'M', u'丁'), + (0x319D, 'M', u'天'), + (0x319E, 'M', u'地'), + (0x319F, 'M', u'人'), + (0x31A0, 'V'), + (0x31E4, 'X'), + (0x31F0, 'V'), + (0x3200, '3', u'(ᄀ)'), + (0x3201, '3', u'(ᄂ)'), + (0x3202, '3', u'(ᄃ)'), + (0x3203, '3', u'(ᄅ)'), + (0x3204, '3', u'(ᄆ)'), + (0x3205, '3', u'(ᄇ)'), + (0x3206, '3', u'(ᄉ)'), + (0x3207, '3', u'(ᄋ)'), + (0x3208, '3', u'(ᄌ)'), + (0x3209, '3', u'(ᄎ)'), + (0x320A, '3', u'(ᄏ)'), + (0x320B, '3', u'(ᄐ)'), + (0x320C, '3', u'(ᄑ)'), + (0x320D, '3', u'(ᄒ)'), + (0x320E, '3', u'(가)'), + (0x320F, '3', u'(나)'), + (0x3210, '3', u'(다)'), + (0x3211, '3', u'(라)'), + (0x3212, '3', u'(마)'), + (0x3213, '3', u'(바)'), + (0x3214, '3', u'(사)'), + (0x3215, '3', u'(아)'), + (0x3216, '3', u'(자)'), + (0x3217, '3', u'(차)'), + (0x3218, '3', u'(카)'), + (0x3219, '3', u'(타)'), + (0x321A, '3', u'(파)'), + (0x321B, '3', u'(하)'), + (0x321C, '3', u'(주)'), + (0x321D, '3', u'(오전)'), + (0x321E, '3', u'(오후)'), + (0x321F, 'X'), + (0x3220, '3', u'(一)'), + (0x3221, '3', u'(二)'), + (0x3222, '3', u'(三)'), + (0x3223, '3', u'(四)'), + (0x3224, '3', u'(五)'), + (0x3225, '3', u'(六)'), + (0x3226, '3', u'(七)'), + (0x3227, '3', u'(八)'), + (0x3228, '3', u'(九)'), + (0x3229, '3', u'(十)'), + (0x322A, '3', u'(月)'), + (0x322B, '3', u'(火)'), + (0x322C, '3', u'(水)'), + (0x322D, '3', u'(木)'), + (0x322E, '3', u'(金)'), + (0x322F, '3', u'(土)'), + (0x3230, '3', u'(日)'), + (0x3231, '3', u'(株)'), + (0x3232, '3', u'(有)'), + (0x3233, '3', u'(社)'), + (0x3234, '3', u'(名)'), + (0x3235, '3', u'(特)'), + (0x3236, '3', u'(財)'), + (0x3237, '3', u'(祝)'), + (0x3238, '3', u'(労)'), + (0x3239, '3', u'(代)'), + (0x323A, '3', u'(呼)'), + (0x323B, '3', u'(学)'), + (0x323C, '3', u'(監)'), + (0x323D, '3', u'(企)'), + (0x323E, '3', u'(資)'), + (0x323F, '3', u'(協)'), + (0x3240, '3', u'(祭)'), + (0x3241, '3', u'(休)'), + (0x3242, '3', u'(自)'), + ] + +def _seg_31(): + return [ + (0x3243, '3', u'(至)'), + (0x3244, 'M', u'問'), + (0x3245, 'M', u'幼'), + (0x3246, 'M', u'文'), + (0x3247, 'M', u'箏'), + (0x3248, 'V'), + (0x3250, 'M', u'pte'), + (0x3251, 'M', u'21'), + (0x3252, 'M', u'22'), + (0x3253, 'M', u'23'), + (0x3254, 'M', u'24'), + (0x3255, 'M', u'25'), + (0x3256, 'M', u'26'), + (0x3257, 'M', u'27'), + (0x3258, 'M', u'28'), + (0x3259, 'M', u'29'), + (0x325A, 'M', u'30'), + (0x325B, 'M', u'31'), + (0x325C, 'M', u'32'), + (0x325D, 'M', u'33'), + (0x325E, 'M', u'34'), + (0x325F, 'M', u'35'), + (0x3260, 'M', u'ᄀ'), + (0x3261, 'M', u'ᄂ'), + (0x3262, 'M', u'ᄃ'), + (0x3263, 'M', u'ᄅ'), + (0x3264, 'M', u'ᄆ'), + (0x3265, 'M', u'ᄇ'), + (0x3266, 'M', u'ᄉ'), + (0x3267, 'M', u'ᄋ'), + (0x3268, 'M', u'ᄌ'), + (0x3269, 'M', u'ᄎ'), + (0x326A, 'M', u'ᄏ'), + (0x326B, 'M', u'ᄐ'), + (0x326C, 'M', u'ᄑ'), + (0x326D, 'M', u'ᄒ'), + (0x326E, 'M', u'가'), + (0x326F, 'M', u'나'), + (0x3270, 'M', u'다'), + (0x3271, 'M', u'라'), + (0x3272, 'M', u'마'), + (0x3273, 'M', u'바'), + (0x3274, 'M', u'사'), + (0x3275, 'M', u'아'), + (0x3276, 'M', u'자'), + (0x3277, 'M', u'차'), + (0x3278, 'M', u'카'), + (0x3279, 'M', u'타'), + (0x327A, 'M', u'파'), + (0x327B, 'M', u'하'), + (0x327C, 'M', u'참고'), + (0x327D, 'M', u'주의'), + (0x327E, 'M', u'우'), + (0x327F, 'V'), + (0x3280, 'M', u'一'), + (0x3281, 'M', u'二'), + (0x3282, 'M', u'三'), + (0x3283, 'M', u'四'), + (0x3284, 'M', u'五'), + (0x3285, 'M', u'六'), + (0x3286, 'M', u'七'), + (0x3287, 'M', u'八'), + (0x3288, 'M', u'九'), + (0x3289, 'M', u'十'), + (0x328A, 'M', u'月'), + (0x328B, 'M', u'火'), + (0x328C, 'M', u'水'), + (0x328D, 'M', u'木'), + (0x328E, 'M', u'金'), + (0x328F, 'M', u'土'), + (0x3290, 'M', u'日'), + (0x3291, 'M', u'株'), + (0x3292, 'M', u'有'), + (0x3293, 'M', u'社'), + (0x3294, 'M', u'名'), + (0x3295, 'M', u'特'), + (0x3296, 'M', u'財'), + (0x3297, 'M', u'祝'), + (0x3298, 'M', u'労'), + (0x3299, 'M', u'秘'), + (0x329A, 'M', u'男'), + (0x329B, 'M', u'女'), + (0x329C, 'M', u'適'), + (0x329D, 'M', u'優'), + (0x329E, 'M', u'印'), + (0x329F, 'M', u'注'), + (0x32A0, 'M', u'項'), + (0x32A1, 'M', u'休'), + (0x32A2, 'M', u'写'), + (0x32A3, 'M', u'正'), + (0x32A4, 'M', u'上'), + (0x32A5, 'M', u'中'), + (0x32A6, 'M', u'下'), + (0x32A7, 'M', u'左'), + (0x32A8, 'M', u'右'), + (0x32A9, 'M', u'医'), + (0x32AA, 'M', u'宗'), + (0x32AB, 'M', u'学'), + (0x32AC, 'M', u'監'), + (0x32AD, 'M', u'企'), + ] + +def _seg_32(): + return [ + (0x32AE, 'M', u'資'), + (0x32AF, 'M', u'協'), + (0x32B0, 'M', u'夜'), + (0x32B1, 'M', u'36'), + (0x32B2, 'M', u'37'), + (0x32B3, 'M', u'38'), + (0x32B4, 'M', u'39'), + (0x32B5, 'M', u'40'), + (0x32B6, 'M', u'41'), + (0x32B7, 'M', u'42'), + (0x32B8, 'M', u'43'), + (0x32B9, 'M', u'44'), + (0x32BA, 'M', u'45'), + (0x32BB, 'M', u'46'), + (0x32BC, 'M', u'47'), + (0x32BD, 'M', u'48'), + (0x32BE, 'M', u'49'), + (0x32BF, 'M', u'50'), + (0x32C0, 'M', u'1月'), + (0x32C1, 'M', u'2月'), + (0x32C2, 'M', u'3月'), + (0x32C3, 'M', u'4月'), + (0x32C4, 'M', u'5月'), + (0x32C5, 'M', u'6月'), + (0x32C6, 'M', u'7月'), + (0x32C7, 'M', u'8月'), + (0x32C8, 'M', u'9月'), + (0x32C9, 'M', u'10月'), + (0x32CA, 'M', u'11月'), + (0x32CB, 'M', u'12月'), + (0x32CC, 'M', u'hg'), + (0x32CD, 'M', u'erg'), + (0x32CE, 'M', u'ev'), + (0x32CF, 'M', u'ltd'), + (0x32D0, 'M', u'ア'), + (0x32D1, 'M', u'イ'), + (0x32D2, 'M', u'ウ'), + (0x32D3, 'M', u'エ'), + (0x32D4, 'M', u'オ'), + (0x32D5, 'M', u'カ'), + (0x32D6, 'M', u'キ'), + (0x32D7, 'M', u'ク'), + (0x32D8, 'M', u'ケ'), + (0x32D9, 'M', u'コ'), + (0x32DA, 'M', u'サ'), + (0x32DB, 'M', u'シ'), + (0x32DC, 'M', u'ス'), + (0x32DD, 'M', u'セ'), + (0x32DE, 'M', u'ソ'), + (0x32DF, 'M', u'タ'), + (0x32E0, 'M', u'チ'), + (0x32E1, 'M', u'ツ'), + (0x32E2, 'M', u'テ'), + (0x32E3, 'M', u'ト'), + (0x32E4, 'M', u'ナ'), + (0x32E5, 'M', u'ニ'), + (0x32E6, 'M', u'ヌ'), + (0x32E7, 'M', u'ネ'), + (0x32E8, 'M', u'ノ'), + (0x32E9, 'M', u'ハ'), + (0x32EA, 'M', u'ヒ'), + (0x32EB, 'M', u'フ'), + (0x32EC, 'M', u'ヘ'), + (0x32ED, 'M', u'ホ'), + (0x32EE, 'M', u'マ'), + (0x32EF, 'M', u'ミ'), + (0x32F0, 'M', u'ム'), + (0x32F1, 'M', u'メ'), + (0x32F2, 'M', u'モ'), + (0x32F3, 'M', u'ヤ'), + (0x32F4, 'M', u'ユ'), + (0x32F5, 'M', u'ヨ'), + (0x32F6, 'M', u'ラ'), + (0x32F7, 'M', u'リ'), + (0x32F8, 'M', u'ル'), + (0x32F9, 'M', u'レ'), + (0x32FA, 'M', u'ロ'), + (0x32FB, 'M', u'ワ'), + (0x32FC, 'M', u'ヰ'), + (0x32FD, 'M', u'ヱ'), + (0x32FE, 'M', u'ヲ'), + (0x32FF, 'M', u'令和'), + (0x3300, 'M', u'アパート'), + (0x3301, 'M', u'アルファ'), + (0x3302, 'M', u'アンペア'), + (0x3303, 'M', u'アール'), + (0x3304, 'M', u'イニング'), + (0x3305, 'M', u'インチ'), + (0x3306, 'M', u'ウォン'), + (0x3307, 'M', u'エスクード'), + (0x3308, 'M', u'エーカー'), + (0x3309, 'M', u'オンス'), + (0x330A, 'M', u'オーム'), + (0x330B, 'M', u'カイリ'), + (0x330C, 'M', u'カラット'), + (0x330D, 'M', u'カロリー'), + (0x330E, 'M', u'ガロン'), + (0x330F, 'M', u'ガンマ'), + (0x3310, 'M', u'ギガ'), + (0x3311, 'M', u'ギニー'), + ] + +def _seg_33(): + return [ + (0x3312, 'M', u'キュリー'), + (0x3313, 'M', u'ギルダー'), + (0x3314, 'M', u'キロ'), + (0x3315, 'M', u'キログラム'), + (0x3316, 'M', u'キロメートル'), + (0x3317, 'M', u'キロワット'), + (0x3318, 'M', u'グラム'), + (0x3319, 'M', u'グラムトン'), + (0x331A, 'M', u'クルゼイロ'), + (0x331B, 'M', u'クローネ'), + (0x331C, 'M', u'ケース'), + (0x331D, 'M', u'コルナ'), + (0x331E, 'M', u'コーポ'), + (0x331F, 'M', u'サイクル'), + (0x3320, 'M', u'サンチーム'), + (0x3321, 'M', u'シリング'), + (0x3322, 'M', u'センチ'), + (0x3323, 'M', u'セント'), + (0x3324, 'M', u'ダース'), + (0x3325, 'M', u'デシ'), + (0x3326, 'M', u'ドル'), + (0x3327, 'M', u'トン'), + (0x3328, 'M', u'ナノ'), + (0x3329, 'M', u'ノット'), + (0x332A, 'M', u'ハイツ'), + (0x332B, 'M', u'パーセント'), + (0x332C, 'M', u'パーツ'), + (0x332D, 'M', u'バーレル'), + (0x332E, 'M', u'ピアストル'), + (0x332F, 'M', u'ピクル'), + (0x3330, 'M', u'ピコ'), + (0x3331, 'M', u'ビル'), + (0x3332, 'M', u'ファラッド'), + (0x3333, 'M', u'フィート'), + (0x3334, 'M', u'ブッシェル'), + (0x3335, 'M', u'フラン'), + (0x3336, 'M', u'ヘクタール'), + (0x3337, 'M', u'ペソ'), + (0x3338, 'M', u'ペニヒ'), + (0x3339, 'M', u'ヘルツ'), + (0x333A, 'M', u'ペンス'), + (0x333B, 'M', u'ページ'), + (0x333C, 'M', u'ベータ'), + (0x333D, 'M', u'ポイント'), + (0x333E, 'M', u'ボルト'), + (0x333F, 'M', u'ホン'), + (0x3340, 'M', u'ポンド'), + (0x3341, 'M', u'ホール'), + (0x3342, 'M', u'ホーン'), + (0x3343, 'M', u'マイクロ'), + (0x3344, 'M', u'マイル'), + (0x3345, 'M', u'マッハ'), + (0x3346, 'M', u'マルク'), + (0x3347, 'M', u'マンション'), + (0x3348, 'M', u'ミクロン'), + (0x3349, 'M', u'ミリ'), + (0x334A, 'M', u'ミリバール'), + (0x334B, 'M', u'メガ'), + (0x334C, 'M', u'メガトン'), + (0x334D, 'M', u'メートル'), + (0x334E, 'M', u'ヤード'), + (0x334F, 'M', u'ヤール'), + (0x3350, 'M', u'ユアン'), + (0x3351, 'M', u'リットル'), + (0x3352, 'M', u'リラ'), + (0x3353, 'M', u'ルピー'), + (0x3354, 'M', u'ルーブル'), + (0x3355, 'M', u'レム'), + (0x3356, 'M', u'レントゲン'), + (0x3357, 'M', u'ワット'), + (0x3358, 'M', u'0点'), + (0x3359, 'M', u'1点'), + (0x335A, 'M', u'2点'), + (0x335B, 'M', u'3点'), + (0x335C, 'M', u'4点'), + (0x335D, 'M', u'5点'), + (0x335E, 'M', u'6点'), + (0x335F, 'M', u'7点'), + (0x3360, 'M', u'8点'), + (0x3361, 'M', u'9点'), + (0x3362, 'M', u'10点'), + (0x3363, 'M', u'11点'), + (0x3364, 'M', u'12点'), + (0x3365, 'M', u'13点'), + (0x3366, 'M', u'14点'), + (0x3367, 'M', u'15点'), + (0x3368, 'M', u'16点'), + (0x3369, 'M', u'17点'), + (0x336A, 'M', u'18点'), + (0x336B, 'M', u'19点'), + (0x336C, 'M', u'20点'), + (0x336D, 'M', u'21点'), + (0x336E, 'M', u'22点'), + (0x336F, 'M', u'23点'), + (0x3370, 'M', u'24点'), + (0x3371, 'M', u'hpa'), + (0x3372, 'M', u'da'), + (0x3373, 'M', u'au'), + (0x3374, 'M', u'bar'), + (0x3375, 'M', u'ov'), + ] + +def _seg_34(): + return [ + (0x3376, 'M', u'pc'), + (0x3377, 'M', u'dm'), + (0x3378, 'M', u'dm2'), + (0x3379, 'M', u'dm3'), + (0x337A, 'M', u'iu'), + (0x337B, 'M', u'平成'), + (0x337C, 'M', u'昭和'), + (0x337D, 'M', u'大正'), + (0x337E, 'M', u'明治'), + (0x337F, 'M', u'株式会社'), + (0x3380, 'M', u'pa'), + (0x3381, 'M', u'na'), + (0x3382, 'M', u'μa'), + (0x3383, 'M', u'ma'), + (0x3384, 'M', u'ka'), + (0x3385, 'M', u'kb'), + (0x3386, 'M', u'mb'), + (0x3387, 'M', u'gb'), + (0x3388, 'M', u'cal'), + (0x3389, 'M', u'kcal'), + (0x338A, 'M', u'pf'), + (0x338B, 'M', u'nf'), + (0x338C, 'M', u'μf'), + (0x338D, 'M', u'μg'), + (0x338E, 'M', u'mg'), + (0x338F, 'M', u'kg'), + (0x3390, 'M', u'hz'), + (0x3391, 'M', u'khz'), + (0x3392, 'M', u'mhz'), + (0x3393, 'M', u'ghz'), + (0x3394, 'M', u'thz'), + (0x3395, 'M', u'μl'), + (0x3396, 'M', u'ml'), + (0x3397, 'M', u'dl'), + (0x3398, 'M', u'kl'), + (0x3399, 'M', u'fm'), + (0x339A, 'M', u'nm'), + (0x339B, 'M', u'μm'), + (0x339C, 'M', u'mm'), + (0x339D, 'M', u'cm'), + (0x339E, 'M', u'km'), + (0x339F, 'M', u'mm2'), + (0x33A0, 'M', u'cm2'), + (0x33A1, 'M', u'm2'), + (0x33A2, 'M', u'km2'), + (0x33A3, 'M', u'mm3'), + (0x33A4, 'M', u'cm3'), + (0x33A5, 'M', u'm3'), + (0x33A6, 'M', u'km3'), + (0x33A7, 'M', u'm∕s'), + (0x33A8, 'M', u'm∕s2'), + (0x33A9, 'M', u'pa'), + (0x33AA, 'M', u'kpa'), + (0x33AB, 'M', u'mpa'), + (0x33AC, 'M', u'gpa'), + (0x33AD, 'M', u'rad'), + (0x33AE, 'M', u'rad∕s'), + (0x33AF, 'M', u'rad∕s2'), + (0x33B0, 'M', u'ps'), + (0x33B1, 'M', u'ns'), + (0x33B2, 'M', u'μs'), + (0x33B3, 'M', u'ms'), + (0x33B4, 'M', u'pv'), + (0x33B5, 'M', u'nv'), + (0x33B6, 'M', u'μv'), + (0x33B7, 'M', u'mv'), + (0x33B8, 'M', u'kv'), + (0x33B9, 'M', u'mv'), + (0x33BA, 'M', u'pw'), + (0x33BB, 'M', u'nw'), + (0x33BC, 'M', u'μw'), + (0x33BD, 'M', u'mw'), + (0x33BE, 'M', u'kw'), + (0x33BF, 'M', u'mw'), + (0x33C0, 'M', u'kω'), + (0x33C1, 'M', u'mω'), + (0x33C2, 'X'), + (0x33C3, 'M', u'bq'), + (0x33C4, 'M', u'cc'), + (0x33C5, 'M', u'cd'), + (0x33C6, 'M', u'c∕kg'), + (0x33C7, 'X'), + (0x33C8, 'M', u'db'), + (0x33C9, 'M', u'gy'), + (0x33CA, 'M', u'ha'), + (0x33CB, 'M', u'hp'), + (0x33CC, 'M', u'in'), + (0x33CD, 'M', u'kk'), + (0x33CE, 'M', u'km'), + (0x33CF, 'M', u'kt'), + (0x33D0, 'M', u'lm'), + (0x33D1, 'M', u'ln'), + (0x33D2, 'M', u'log'), + (0x33D3, 'M', u'lx'), + (0x33D4, 'M', u'mb'), + (0x33D5, 'M', u'mil'), + (0x33D6, 'M', u'mol'), + (0x33D7, 'M', u'ph'), + (0x33D8, 'X'), + (0x33D9, 'M', u'ppm'), + ] + +def _seg_35(): + return [ + (0x33DA, 'M', u'pr'), + (0x33DB, 'M', u'sr'), + (0x33DC, 'M', u'sv'), + (0x33DD, 'M', u'wb'), + (0x33DE, 'M', u'v∕m'), + (0x33DF, 'M', u'a∕m'), + (0x33E0, 'M', u'1日'), + (0x33E1, 'M', u'2日'), + (0x33E2, 'M', u'3日'), + (0x33E3, 'M', u'4日'), + (0x33E4, 'M', u'5日'), + (0x33E5, 'M', u'6日'), + (0x33E6, 'M', u'7日'), + (0x33E7, 'M', u'8日'), + (0x33E8, 'M', u'9日'), + (0x33E9, 'M', u'10日'), + (0x33EA, 'M', u'11日'), + (0x33EB, 'M', u'12日'), + (0x33EC, 'M', u'13日'), + (0x33ED, 'M', u'14日'), + (0x33EE, 'M', u'15日'), + (0x33EF, 'M', u'16日'), + (0x33F0, 'M', u'17日'), + (0x33F1, 'M', u'18日'), + (0x33F2, 'M', u'19日'), + (0x33F3, 'M', u'20日'), + (0x33F4, 'M', u'21日'), + (0x33F5, 'M', u'22日'), + (0x33F6, 'M', u'23日'), + (0x33F7, 'M', u'24日'), + (0x33F8, 'M', u'25日'), + (0x33F9, 'M', u'26日'), + (0x33FA, 'M', u'27日'), + (0x33FB, 'M', u'28日'), + (0x33FC, 'M', u'29日'), + (0x33FD, 'M', u'30日'), + (0x33FE, 'M', u'31日'), + (0x33FF, 'M', u'gal'), + (0x3400, 'V'), + (0x9FFD, 'X'), + (0xA000, 'V'), + (0xA48D, 'X'), + (0xA490, 'V'), + (0xA4C7, 'X'), + (0xA4D0, 'V'), + (0xA62C, 'X'), + (0xA640, 'M', u'ꙁ'), + (0xA641, 'V'), + (0xA642, 'M', u'ꙃ'), + (0xA643, 'V'), + (0xA644, 'M', u'ꙅ'), + (0xA645, 'V'), + (0xA646, 'M', u'ꙇ'), + (0xA647, 'V'), + (0xA648, 'M', u'ꙉ'), + (0xA649, 'V'), + (0xA64A, 'M', u'ꙋ'), + (0xA64B, 'V'), + (0xA64C, 'M', u'ꙍ'), + (0xA64D, 'V'), + (0xA64E, 'M', u'ꙏ'), + (0xA64F, 'V'), + (0xA650, 'M', u'ꙑ'), + (0xA651, 'V'), + (0xA652, 'M', u'ꙓ'), + (0xA653, 'V'), + (0xA654, 'M', u'ꙕ'), + (0xA655, 'V'), + (0xA656, 'M', u'ꙗ'), + (0xA657, 'V'), + (0xA658, 'M', u'ꙙ'), + (0xA659, 'V'), + (0xA65A, 'M', u'ꙛ'), + (0xA65B, 'V'), + (0xA65C, 'M', u'ꙝ'), + (0xA65D, 'V'), + (0xA65E, 'M', u'ꙟ'), + (0xA65F, 'V'), + (0xA660, 'M', u'ꙡ'), + (0xA661, 'V'), + (0xA662, 'M', u'ꙣ'), + (0xA663, 'V'), + (0xA664, 'M', u'ꙥ'), + (0xA665, 'V'), + (0xA666, 'M', u'ꙧ'), + (0xA667, 'V'), + (0xA668, 'M', u'ꙩ'), + (0xA669, 'V'), + (0xA66A, 'M', u'ꙫ'), + (0xA66B, 'V'), + (0xA66C, 'M', u'ꙭ'), + (0xA66D, 'V'), + (0xA680, 'M', u'ꚁ'), + (0xA681, 'V'), + (0xA682, 'M', u'ꚃ'), + (0xA683, 'V'), + (0xA684, 'M', u'ꚅ'), + (0xA685, 'V'), + (0xA686, 'M', u'ꚇ'), + (0xA687, 'V'), + ] + +def _seg_36(): + return [ + (0xA688, 'M', u'ꚉ'), + (0xA689, 'V'), + (0xA68A, 'M', u'ꚋ'), + (0xA68B, 'V'), + (0xA68C, 'M', u'ꚍ'), + (0xA68D, 'V'), + (0xA68E, 'M', u'ꚏ'), + (0xA68F, 'V'), + (0xA690, 'M', u'ꚑ'), + (0xA691, 'V'), + (0xA692, 'M', u'ꚓ'), + (0xA693, 'V'), + (0xA694, 'M', u'ꚕ'), + (0xA695, 'V'), + (0xA696, 'M', u'ꚗ'), + (0xA697, 'V'), + (0xA698, 'M', u'ꚙ'), + (0xA699, 'V'), + (0xA69A, 'M', u'ꚛ'), + (0xA69B, 'V'), + (0xA69C, 'M', u'ъ'), + (0xA69D, 'M', u'ь'), + (0xA69E, 'V'), + (0xA6F8, 'X'), + (0xA700, 'V'), + (0xA722, 'M', u'ꜣ'), + (0xA723, 'V'), + (0xA724, 'M', u'ꜥ'), + (0xA725, 'V'), + (0xA726, 'M', u'ꜧ'), + (0xA727, 'V'), + (0xA728, 'M', u'ꜩ'), + (0xA729, 'V'), + (0xA72A, 'M', u'ꜫ'), + (0xA72B, 'V'), + (0xA72C, 'M', u'ꜭ'), + (0xA72D, 'V'), + (0xA72E, 'M', u'ꜯ'), + (0xA72F, 'V'), + (0xA732, 'M', u'ꜳ'), + (0xA733, 'V'), + (0xA734, 'M', u'ꜵ'), + (0xA735, 'V'), + (0xA736, 'M', u'ꜷ'), + (0xA737, 'V'), + (0xA738, 'M', u'ꜹ'), + (0xA739, 'V'), + (0xA73A, 'M', u'ꜻ'), + (0xA73B, 'V'), + (0xA73C, 'M', u'ꜽ'), + (0xA73D, 'V'), + (0xA73E, 'M', u'ꜿ'), + (0xA73F, 'V'), + (0xA740, 'M', u'ꝁ'), + (0xA741, 'V'), + (0xA742, 'M', u'ꝃ'), + (0xA743, 'V'), + (0xA744, 'M', u'ꝅ'), + (0xA745, 'V'), + (0xA746, 'M', u'ꝇ'), + (0xA747, 'V'), + (0xA748, 'M', u'ꝉ'), + (0xA749, 'V'), + (0xA74A, 'M', u'ꝋ'), + (0xA74B, 'V'), + (0xA74C, 'M', u'ꝍ'), + (0xA74D, 'V'), + (0xA74E, 'M', u'ꝏ'), + (0xA74F, 'V'), + (0xA750, 'M', u'ꝑ'), + (0xA751, 'V'), + (0xA752, 'M', u'ꝓ'), + (0xA753, 'V'), + (0xA754, 'M', u'ꝕ'), + (0xA755, 'V'), + (0xA756, 'M', u'ꝗ'), + (0xA757, 'V'), + (0xA758, 'M', u'ꝙ'), + (0xA759, 'V'), + (0xA75A, 'M', u'ꝛ'), + (0xA75B, 'V'), + (0xA75C, 'M', u'ꝝ'), + (0xA75D, 'V'), + (0xA75E, 'M', u'ꝟ'), + (0xA75F, 'V'), + (0xA760, 'M', u'ꝡ'), + (0xA761, 'V'), + (0xA762, 'M', u'ꝣ'), + (0xA763, 'V'), + (0xA764, 'M', u'ꝥ'), + (0xA765, 'V'), + (0xA766, 'M', u'ꝧ'), + (0xA767, 'V'), + (0xA768, 'M', u'ꝩ'), + (0xA769, 'V'), + (0xA76A, 'M', u'ꝫ'), + (0xA76B, 'V'), + (0xA76C, 'M', u'ꝭ'), + (0xA76D, 'V'), + (0xA76E, 'M', u'ꝯ'), + ] + +def _seg_37(): + return [ + (0xA76F, 'V'), + (0xA770, 'M', u'ꝯ'), + (0xA771, 'V'), + (0xA779, 'M', u'ꝺ'), + (0xA77A, 'V'), + (0xA77B, 'M', u'ꝼ'), + (0xA77C, 'V'), + (0xA77D, 'M', u'ᵹ'), + (0xA77E, 'M', u'ꝿ'), + (0xA77F, 'V'), + (0xA780, 'M', u'ꞁ'), + (0xA781, 'V'), + (0xA782, 'M', u'ꞃ'), + (0xA783, 'V'), + (0xA784, 'M', u'ꞅ'), + (0xA785, 'V'), + (0xA786, 'M', u'ꞇ'), + (0xA787, 'V'), + (0xA78B, 'M', u'ꞌ'), + (0xA78C, 'V'), + (0xA78D, 'M', u'ɥ'), + (0xA78E, 'V'), + (0xA790, 'M', u'ꞑ'), + (0xA791, 'V'), + (0xA792, 'M', u'ꞓ'), + (0xA793, 'V'), + (0xA796, 'M', u'ꞗ'), + (0xA797, 'V'), + (0xA798, 'M', u'ꞙ'), + (0xA799, 'V'), + (0xA79A, 'M', u'ꞛ'), + (0xA79B, 'V'), + (0xA79C, 'M', u'ꞝ'), + (0xA79D, 'V'), + (0xA79E, 'M', u'ꞟ'), + (0xA79F, 'V'), + (0xA7A0, 'M', u'ꞡ'), + (0xA7A1, 'V'), + (0xA7A2, 'M', u'ꞣ'), + (0xA7A3, 'V'), + (0xA7A4, 'M', u'ꞥ'), + (0xA7A5, 'V'), + (0xA7A6, 'M', u'ꞧ'), + (0xA7A7, 'V'), + (0xA7A8, 'M', u'ꞩ'), + (0xA7A9, 'V'), + (0xA7AA, 'M', u'ɦ'), + (0xA7AB, 'M', u'ɜ'), + (0xA7AC, 'M', u'ɡ'), + (0xA7AD, 'M', u'ɬ'), + (0xA7AE, 'M', u'ɪ'), + (0xA7AF, 'V'), + (0xA7B0, 'M', u'ʞ'), + (0xA7B1, 'M', u'ʇ'), + (0xA7B2, 'M', u'ʝ'), + (0xA7B3, 'M', u'ꭓ'), + (0xA7B4, 'M', u'ꞵ'), + (0xA7B5, 'V'), + (0xA7B6, 'M', u'ꞷ'), + (0xA7B7, 'V'), + (0xA7B8, 'M', u'ꞹ'), + (0xA7B9, 'V'), + (0xA7BA, 'M', u'ꞻ'), + (0xA7BB, 'V'), + (0xA7BC, 'M', u'ꞽ'), + (0xA7BD, 'V'), + (0xA7BE, 'M', u'ꞿ'), + (0xA7BF, 'V'), + (0xA7C0, 'X'), + (0xA7C2, 'M', u'ꟃ'), + (0xA7C3, 'V'), + (0xA7C4, 'M', u'ꞔ'), + (0xA7C5, 'M', u'ʂ'), + (0xA7C6, 'M', u'ᶎ'), + (0xA7C7, 'M', u'ꟈ'), + (0xA7C8, 'V'), + (0xA7C9, 'M', u'ꟊ'), + (0xA7CA, 'V'), + (0xA7CB, 'X'), + (0xA7F5, 'M', u'ꟶ'), + (0xA7F6, 'V'), + (0xA7F8, 'M', u'ħ'), + (0xA7F9, 'M', u'œ'), + (0xA7FA, 'V'), + (0xA82D, 'X'), + (0xA830, 'V'), + (0xA83A, 'X'), + (0xA840, 'V'), + (0xA878, 'X'), + (0xA880, 'V'), + (0xA8C6, 'X'), + (0xA8CE, 'V'), + (0xA8DA, 'X'), + (0xA8E0, 'V'), + (0xA954, 'X'), + (0xA95F, 'V'), + (0xA97D, 'X'), + (0xA980, 'V'), + (0xA9CE, 'X'), + (0xA9CF, 'V'), + ] + +def _seg_38(): + return [ + (0xA9DA, 'X'), + (0xA9DE, 'V'), + (0xA9FF, 'X'), + (0xAA00, 'V'), + (0xAA37, 'X'), + (0xAA40, 'V'), + (0xAA4E, 'X'), + (0xAA50, 'V'), + (0xAA5A, 'X'), + (0xAA5C, 'V'), + (0xAAC3, 'X'), + (0xAADB, 'V'), + (0xAAF7, 'X'), + (0xAB01, 'V'), + (0xAB07, 'X'), + (0xAB09, 'V'), + (0xAB0F, 'X'), + (0xAB11, 'V'), + (0xAB17, 'X'), + (0xAB20, 'V'), + (0xAB27, 'X'), + (0xAB28, 'V'), + (0xAB2F, 'X'), + (0xAB30, 'V'), + (0xAB5C, 'M', u'ꜧ'), + (0xAB5D, 'M', u'ꬷ'), + (0xAB5E, 'M', u'ɫ'), + (0xAB5F, 'M', u'ꭒ'), + (0xAB60, 'V'), + (0xAB69, 'M', u'ʍ'), + (0xAB6A, 'V'), + (0xAB6C, 'X'), + (0xAB70, 'M', u'Ꭰ'), + (0xAB71, 'M', u'Ꭱ'), + (0xAB72, 'M', u'Ꭲ'), + (0xAB73, 'M', u'Ꭳ'), + (0xAB74, 'M', u'Ꭴ'), + (0xAB75, 'M', u'Ꭵ'), + (0xAB76, 'M', u'Ꭶ'), + (0xAB77, 'M', u'Ꭷ'), + (0xAB78, 'M', u'Ꭸ'), + (0xAB79, 'M', u'Ꭹ'), + (0xAB7A, 'M', u'Ꭺ'), + (0xAB7B, 'M', u'Ꭻ'), + (0xAB7C, 'M', u'Ꭼ'), + (0xAB7D, 'M', u'Ꭽ'), + (0xAB7E, 'M', u'Ꭾ'), + (0xAB7F, 'M', u'Ꭿ'), + (0xAB80, 'M', u'Ꮀ'), + (0xAB81, 'M', u'Ꮁ'), + (0xAB82, 'M', u'Ꮂ'), + (0xAB83, 'M', u'Ꮃ'), + (0xAB84, 'M', u'Ꮄ'), + (0xAB85, 'M', u'Ꮅ'), + (0xAB86, 'M', u'Ꮆ'), + (0xAB87, 'M', u'Ꮇ'), + (0xAB88, 'M', u'Ꮈ'), + (0xAB89, 'M', u'Ꮉ'), + (0xAB8A, 'M', u'Ꮊ'), + (0xAB8B, 'M', u'Ꮋ'), + (0xAB8C, 'M', u'Ꮌ'), + (0xAB8D, 'M', u'Ꮍ'), + (0xAB8E, 'M', u'Ꮎ'), + (0xAB8F, 'M', u'Ꮏ'), + (0xAB90, 'M', u'Ꮐ'), + (0xAB91, 'M', u'Ꮑ'), + (0xAB92, 'M', u'Ꮒ'), + (0xAB93, 'M', u'Ꮓ'), + (0xAB94, 'M', u'Ꮔ'), + (0xAB95, 'M', u'Ꮕ'), + (0xAB96, 'M', u'Ꮖ'), + (0xAB97, 'M', u'Ꮗ'), + (0xAB98, 'M', u'Ꮘ'), + (0xAB99, 'M', u'Ꮙ'), + (0xAB9A, 'M', u'Ꮚ'), + (0xAB9B, 'M', u'Ꮛ'), + (0xAB9C, 'M', u'Ꮜ'), + (0xAB9D, 'M', u'Ꮝ'), + (0xAB9E, 'M', u'Ꮞ'), + (0xAB9F, 'M', u'Ꮟ'), + (0xABA0, 'M', u'Ꮠ'), + (0xABA1, 'M', u'Ꮡ'), + (0xABA2, 'M', u'Ꮢ'), + (0xABA3, 'M', u'Ꮣ'), + (0xABA4, 'M', u'Ꮤ'), + (0xABA5, 'M', u'Ꮥ'), + (0xABA6, 'M', u'Ꮦ'), + (0xABA7, 'M', u'Ꮧ'), + (0xABA8, 'M', u'Ꮨ'), + (0xABA9, 'M', u'Ꮩ'), + (0xABAA, 'M', u'Ꮪ'), + (0xABAB, 'M', u'Ꮫ'), + (0xABAC, 'M', u'Ꮬ'), + (0xABAD, 'M', u'Ꮭ'), + (0xABAE, 'M', u'Ꮮ'), + (0xABAF, 'M', u'Ꮯ'), + (0xABB0, 'M', u'Ꮰ'), + (0xABB1, 'M', u'Ꮱ'), + (0xABB2, 'M', u'Ꮲ'), + (0xABB3, 'M', u'Ꮳ'), + ] + +def _seg_39(): + return [ + (0xABB4, 'M', u'Ꮴ'), + (0xABB5, 'M', u'Ꮵ'), + (0xABB6, 'M', u'Ꮶ'), + (0xABB7, 'M', u'Ꮷ'), + (0xABB8, 'M', u'Ꮸ'), + (0xABB9, 'M', u'Ꮹ'), + (0xABBA, 'M', u'Ꮺ'), + (0xABBB, 'M', u'Ꮻ'), + (0xABBC, 'M', u'Ꮼ'), + (0xABBD, 'M', u'Ꮽ'), + (0xABBE, 'M', u'Ꮾ'), + (0xABBF, 'M', u'Ꮿ'), + (0xABC0, 'V'), + (0xABEE, 'X'), + (0xABF0, 'V'), + (0xABFA, 'X'), + (0xAC00, 'V'), + (0xD7A4, 'X'), + (0xD7B0, 'V'), + (0xD7C7, 'X'), + (0xD7CB, 'V'), + (0xD7FC, 'X'), + (0xF900, 'M', u'豈'), + (0xF901, 'M', u'更'), + (0xF902, 'M', u'車'), + (0xF903, 'M', u'賈'), + (0xF904, 'M', u'滑'), + (0xF905, 'M', u'串'), + (0xF906, 'M', u'句'), + (0xF907, 'M', u'龜'), + (0xF909, 'M', u'契'), + (0xF90A, 'M', u'金'), + (0xF90B, 'M', u'喇'), + (0xF90C, 'M', u'奈'), + (0xF90D, 'M', u'懶'), + (0xF90E, 'M', u'癩'), + (0xF90F, 'M', u'羅'), + (0xF910, 'M', u'蘿'), + (0xF911, 'M', u'螺'), + (0xF912, 'M', u'裸'), + (0xF913, 'M', u'邏'), + (0xF914, 'M', u'樂'), + (0xF915, 'M', u'洛'), + (0xF916, 'M', u'烙'), + (0xF917, 'M', u'珞'), + (0xF918, 'M', u'落'), + (0xF919, 'M', u'酪'), + (0xF91A, 'M', u'駱'), + (0xF91B, 'M', u'亂'), + (0xF91C, 'M', u'卵'), + (0xF91D, 'M', u'欄'), + (0xF91E, 'M', u'爛'), + (0xF91F, 'M', u'蘭'), + (0xF920, 'M', u'鸞'), + (0xF921, 'M', u'嵐'), + (0xF922, 'M', u'濫'), + (0xF923, 'M', u'藍'), + (0xF924, 'M', u'襤'), + (0xF925, 'M', u'拉'), + (0xF926, 'M', u'臘'), + (0xF927, 'M', u'蠟'), + (0xF928, 'M', u'廊'), + (0xF929, 'M', u'朗'), + (0xF92A, 'M', u'浪'), + (0xF92B, 'M', u'狼'), + (0xF92C, 'M', u'郎'), + (0xF92D, 'M', u'來'), + (0xF92E, 'M', u'冷'), + (0xF92F, 'M', u'勞'), + (0xF930, 'M', u'擄'), + (0xF931, 'M', u'櫓'), + (0xF932, 'M', u'爐'), + (0xF933, 'M', u'盧'), + (0xF934, 'M', u'老'), + (0xF935, 'M', u'蘆'), + (0xF936, 'M', u'虜'), + (0xF937, 'M', u'路'), + (0xF938, 'M', u'露'), + (0xF939, 'M', u'魯'), + (0xF93A, 'M', u'鷺'), + (0xF93B, 'M', u'碌'), + (0xF93C, 'M', u'祿'), + (0xF93D, 'M', u'綠'), + (0xF93E, 'M', u'菉'), + (0xF93F, 'M', u'錄'), + (0xF940, 'M', u'鹿'), + (0xF941, 'M', u'論'), + (0xF942, 'M', u'壟'), + (0xF943, 'M', u'弄'), + (0xF944, 'M', u'籠'), + (0xF945, 'M', u'聾'), + (0xF946, 'M', u'牢'), + (0xF947, 'M', u'磊'), + (0xF948, 'M', u'賂'), + (0xF949, 'M', u'雷'), + (0xF94A, 'M', u'壘'), + (0xF94B, 'M', u'屢'), + (0xF94C, 'M', u'樓'), + (0xF94D, 'M', u'淚'), + (0xF94E, 'M', u'漏'), + ] + +def _seg_40(): + return [ + (0xF94F, 'M', u'累'), + (0xF950, 'M', u'縷'), + (0xF951, 'M', u'陋'), + (0xF952, 'M', u'勒'), + (0xF953, 'M', u'肋'), + (0xF954, 'M', u'凜'), + (0xF955, 'M', u'凌'), + (0xF956, 'M', u'稜'), + (0xF957, 'M', u'綾'), + (0xF958, 'M', u'菱'), + (0xF959, 'M', u'陵'), + (0xF95A, 'M', u'讀'), + (0xF95B, 'M', u'拏'), + (0xF95C, 'M', u'樂'), + (0xF95D, 'M', u'諾'), + (0xF95E, 'M', u'丹'), + (0xF95F, 'M', u'寧'), + (0xF960, 'M', u'怒'), + (0xF961, 'M', u'率'), + (0xF962, 'M', u'異'), + (0xF963, 'M', u'北'), + (0xF964, 'M', u'磻'), + (0xF965, 'M', u'便'), + (0xF966, 'M', u'復'), + (0xF967, 'M', u'不'), + (0xF968, 'M', u'泌'), + (0xF969, 'M', u'數'), + (0xF96A, 'M', u'索'), + (0xF96B, 'M', u'參'), + (0xF96C, 'M', u'塞'), + (0xF96D, 'M', u'省'), + (0xF96E, 'M', u'葉'), + (0xF96F, 'M', u'說'), + (0xF970, 'M', u'殺'), + (0xF971, 'M', u'辰'), + (0xF972, 'M', u'沈'), + (0xF973, 'M', u'拾'), + (0xF974, 'M', u'若'), + (0xF975, 'M', u'掠'), + (0xF976, 'M', u'略'), + (0xF977, 'M', u'亮'), + (0xF978, 'M', u'兩'), + (0xF979, 'M', u'凉'), + (0xF97A, 'M', u'梁'), + (0xF97B, 'M', u'糧'), + (0xF97C, 'M', u'良'), + (0xF97D, 'M', u'諒'), + (0xF97E, 'M', u'量'), + (0xF97F, 'M', u'勵'), + (0xF980, 'M', u'呂'), + (0xF981, 'M', u'女'), + (0xF982, 'M', u'廬'), + (0xF983, 'M', u'旅'), + (0xF984, 'M', u'濾'), + (0xF985, 'M', u'礪'), + (0xF986, 'M', u'閭'), + (0xF987, 'M', u'驪'), + (0xF988, 'M', u'麗'), + (0xF989, 'M', u'黎'), + (0xF98A, 'M', u'力'), + (0xF98B, 'M', u'曆'), + (0xF98C, 'M', u'歷'), + (0xF98D, 'M', u'轢'), + (0xF98E, 'M', u'年'), + (0xF98F, 'M', u'憐'), + (0xF990, 'M', u'戀'), + (0xF991, 'M', u'撚'), + (0xF992, 'M', u'漣'), + (0xF993, 'M', u'煉'), + (0xF994, 'M', u'璉'), + (0xF995, 'M', u'秊'), + (0xF996, 'M', u'練'), + (0xF997, 'M', u'聯'), + (0xF998, 'M', u'輦'), + (0xF999, 'M', u'蓮'), + (0xF99A, 'M', u'連'), + (0xF99B, 'M', u'鍊'), + (0xF99C, 'M', u'列'), + (0xF99D, 'M', u'劣'), + (0xF99E, 'M', u'咽'), + (0xF99F, 'M', u'烈'), + (0xF9A0, 'M', u'裂'), + (0xF9A1, 'M', u'說'), + (0xF9A2, 'M', u'廉'), + (0xF9A3, 'M', u'念'), + (0xF9A4, 'M', u'捻'), + (0xF9A5, 'M', u'殮'), + (0xF9A6, 'M', u'簾'), + (0xF9A7, 'M', u'獵'), + (0xF9A8, 'M', u'令'), + (0xF9A9, 'M', u'囹'), + (0xF9AA, 'M', u'寧'), + (0xF9AB, 'M', u'嶺'), + (0xF9AC, 'M', u'怜'), + (0xF9AD, 'M', u'玲'), + (0xF9AE, 'M', u'瑩'), + (0xF9AF, 'M', u'羚'), + (0xF9B0, 'M', u'聆'), + (0xF9B1, 'M', u'鈴'), + (0xF9B2, 'M', u'零'), + ] + +def _seg_41(): + return [ + (0xF9B3, 'M', u'靈'), + (0xF9B4, 'M', u'領'), + (0xF9B5, 'M', u'例'), + (0xF9B6, 'M', u'禮'), + (0xF9B7, 'M', u'醴'), + (0xF9B8, 'M', u'隸'), + (0xF9B9, 'M', u'惡'), + (0xF9BA, 'M', u'了'), + (0xF9BB, 'M', u'僚'), + (0xF9BC, 'M', u'寮'), + (0xF9BD, 'M', u'尿'), + (0xF9BE, 'M', u'料'), + (0xF9BF, 'M', u'樂'), + (0xF9C0, 'M', u'燎'), + (0xF9C1, 'M', u'療'), + (0xF9C2, 'M', u'蓼'), + (0xF9C3, 'M', u'遼'), + (0xF9C4, 'M', u'龍'), + (0xF9C5, 'M', u'暈'), + (0xF9C6, 'M', u'阮'), + (0xF9C7, 'M', u'劉'), + (0xF9C8, 'M', u'杻'), + (0xF9C9, 'M', u'柳'), + (0xF9CA, 'M', u'流'), + (0xF9CB, 'M', u'溜'), + (0xF9CC, 'M', u'琉'), + (0xF9CD, 'M', u'留'), + (0xF9CE, 'M', u'硫'), + (0xF9CF, 'M', u'紐'), + (0xF9D0, 'M', u'類'), + (0xF9D1, 'M', u'六'), + (0xF9D2, 'M', u'戮'), + (0xF9D3, 'M', u'陸'), + (0xF9D4, 'M', u'倫'), + (0xF9D5, 'M', u'崙'), + (0xF9D6, 'M', u'淪'), + (0xF9D7, 'M', u'輪'), + (0xF9D8, 'M', u'律'), + (0xF9D9, 'M', u'慄'), + (0xF9DA, 'M', u'栗'), + (0xF9DB, 'M', u'率'), + (0xF9DC, 'M', u'隆'), + (0xF9DD, 'M', u'利'), + (0xF9DE, 'M', u'吏'), + (0xF9DF, 'M', u'履'), + (0xF9E0, 'M', u'易'), + (0xF9E1, 'M', u'李'), + (0xF9E2, 'M', u'梨'), + (0xF9E3, 'M', u'泥'), + (0xF9E4, 'M', u'理'), + (0xF9E5, 'M', u'痢'), + (0xF9E6, 'M', u'罹'), + (0xF9E7, 'M', u'裏'), + (0xF9E8, 'M', u'裡'), + (0xF9E9, 'M', u'里'), + (0xF9EA, 'M', u'離'), + (0xF9EB, 'M', u'匿'), + (0xF9EC, 'M', u'溺'), + (0xF9ED, 'M', u'吝'), + (0xF9EE, 'M', u'燐'), + (0xF9EF, 'M', u'璘'), + (0xF9F0, 'M', u'藺'), + (0xF9F1, 'M', u'隣'), + (0xF9F2, 'M', u'鱗'), + (0xF9F3, 'M', u'麟'), + (0xF9F4, 'M', u'林'), + (0xF9F5, 'M', u'淋'), + (0xF9F6, 'M', u'臨'), + (0xF9F7, 'M', u'立'), + (0xF9F8, 'M', u'笠'), + (0xF9F9, 'M', u'粒'), + (0xF9FA, 'M', u'狀'), + (0xF9FB, 'M', u'炙'), + (0xF9FC, 'M', u'識'), + (0xF9FD, 'M', u'什'), + (0xF9FE, 'M', u'茶'), + (0xF9FF, 'M', u'刺'), + (0xFA00, 'M', u'切'), + (0xFA01, 'M', u'度'), + (0xFA02, 'M', u'拓'), + (0xFA03, 'M', u'糖'), + (0xFA04, 'M', u'宅'), + (0xFA05, 'M', u'洞'), + (0xFA06, 'M', u'暴'), + (0xFA07, 'M', u'輻'), + (0xFA08, 'M', u'行'), + (0xFA09, 'M', u'降'), + (0xFA0A, 'M', u'見'), + (0xFA0B, 'M', u'廓'), + (0xFA0C, 'M', u'兀'), + (0xFA0D, 'M', u'嗀'), + (0xFA0E, 'V'), + (0xFA10, 'M', u'塚'), + (0xFA11, 'V'), + (0xFA12, 'M', u'晴'), + (0xFA13, 'V'), + (0xFA15, 'M', u'凞'), + (0xFA16, 'M', u'猪'), + (0xFA17, 'M', u'益'), + (0xFA18, 'M', u'礼'), + ] + +def _seg_42(): + return [ + (0xFA19, 'M', u'神'), + (0xFA1A, 'M', u'祥'), + (0xFA1B, 'M', u'福'), + (0xFA1C, 'M', u'靖'), + (0xFA1D, 'M', u'精'), + (0xFA1E, 'M', u'羽'), + (0xFA1F, 'V'), + (0xFA20, 'M', u'蘒'), + (0xFA21, 'V'), + (0xFA22, 'M', u'諸'), + (0xFA23, 'V'), + (0xFA25, 'M', u'逸'), + (0xFA26, 'M', u'都'), + (0xFA27, 'V'), + (0xFA2A, 'M', u'飯'), + (0xFA2B, 'M', u'飼'), + (0xFA2C, 'M', u'館'), + (0xFA2D, 'M', u'鶴'), + (0xFA2E, 'M', u'郞'), + (0xFA2F, 'M', u'隷'), + (0xFA30, 'M', u'侮'), + (0xFA31, 'M', u'僧'), + (0xFA32, 'M', u'免'), + (0xFA33, 'M', u'勉'), + (0xFA34, 'M', u'勤'), + (0xFA35, 'M', u'卑'), + (0xFA36, 'M', u'喝'), + (0xFA37, 'M', u'嘆'), + (0xFA38, 'M', u'器'), + (0xFA39, 'M', u'塀'), + (0xFA3A, 'M', u'墨'), + (0xFA3B, 'M', u'層'), + (0xFA3C, 'M', u'屮'), + (0xFA3D, 'M', u'悔'), + (0xFA3E, 'M', u'慨'), + (0xFA3F, 'M', u'憎'), + (0xFA40, 'M', u'懲'), + (0xFA41, 'M', u'敏'), + (0xFA42, 'M', u'既'), + (0xFA43, 'M', u'暑'), + (0xFA44, 'M', u'梅'), + (0xFA45, 'M', u'海'), + (0xFA46, 'M', u'渚'), + (0xFA47, 'M', u'漢'), + (0xFA48, 'M', u'煮'), + (0xFA49, 'M', u'爫'), + (0xFA4A, 'M', u'琢'), + (0xFA4B, 'M', u'碑'), + (0xFA4C, 'M', u'社'), + (0xFA4D, 'M', u'祉'), + (0xFA4E, 'M', u'祈'), + (0xFA4F, 'M', u'祐'), + (0xFA50, 'M', u'祖'), + (0xFA51, 'M', u'祝'), + (0xFA52, 'M', u'禍'), + (0xFA53, 'M', u'禎'), + (0xFA54, 'M', u'穀'), + (0xFA55, 'M', u'突'), + (0xFA56, 'M', u'節'), + (0xFA57, 'M', u'練'), + (0xFA58, 'M', u'縉'), + (0xFA59, 'M', u'繁'), + (0xFA5A, 'M', u'署'), + (0xFA5B, 'M', u'者'), + (0xFA5C, 'M', u'臭'), + (0xFA5D, 'M', u'艹'), + (0xFA5F, 'M', u'著'), + (0xFA60, 'M', u'褐'), + (0xFA61, 'M', u'視'), + (0xFA62, 'M', u'謁'), + (0xFA63, 'M', u'謹'), + (0xFA64, 'M', u'賓'), + (0xFA65, 'M', u'贈'), + (0xFA66, 'M', u'辶'), + (0xFA67, 'M', u'逸'), + (0xFA68, 'M', u'難'), + (0xFA69, 'M', u'響'), + (0xFA6A, 'M', u'頻'), + (0xFA6B, 'M', u'恵'), + (0xFA6C, 'M', u'𤋮'), + (0xFA6D, 'M', u'舘'), + (0xFA6E, 'X'), + (0xFA70, 'M', u'並'), + (0xFA71, 'M', u'况'), + (0xFA72, 'M', u'全'), + (0xFA73, 'M', u'侀'), + (0xFA74, 'M', u'充'), + (0xFA75, 'M', u'冀'), + (0xFA76, 'M', u'勇'), + (0xFA77, 'M', u'勺'), + (0xFA78, 'M', u'喝'), + (0xFA79, 'M', u'啕'), + (0xFA7A, 'M', u'喙'), + (0xFA7B, 'M', u'嗢'), + (0xFA7C, 'M', u'塚'), + (0xFA7D, 'M', u'墳'), + (0xFA7E, 'M', u'奄'), + (0xFA7F, 'M', u'奔'), + (0xFA80, 'M', u'婢'), + (0xFA81, 'M', u'嬨'), + ] + +def _seg_43(): + return [ + (0xFA82, 'M', u'廒'), + (0xFA83, 'M', u'廙'), + (0xFA84, 'M', u'彩'), + (0xFA85, 'M', u'徭'), + (0xFA86, 'M', u'惘'), + (0xFA87, 'M', u'慎'), + (0xFA88, 'M', u'愈'), + (0xFA89, 'M', u'憎'), + (0xFA8A, 'M', u'慠'), + (0xFA8B, 'M', u'懲'), + (0xFA8C, 'M', u'戴'), + (0xFA8D, 'M', u'揄'), + (0xFA8E, 'M', u'搜'), + (0xFA8F, 'M', u'摒'), + (0xFA90, 'M', u'敖'), + (0xFA91, 'M', u'晴'), + (0xFA92, 'M', u'朗'), + (0xFA93, 'M', u'望'), + (0xFA94, 'M', u'杖'), + (0xFA95, 'M', u'歹'), + (0xFA96, 'M', u'殺'), + (0xFA97, 'M', u'流'), + (0xFA98, 'M', u'滛'), + (0xFA99, 'M', u'滋'), + (0xFA9A, 'M', u'漢'), + (0xFA9B, 'M', u'瀞'), + (0xFA9C, 'M', u'煮'), + (0xFA9D, 'M', u'瞧'), + (0xFA9E, 'M', u'爵'), + (0xFA9F, 'M', u'犯'), + (0xFAA0, 'M', u'猪'), + (0xFAA1, 'M', u'瑱'), + (0xFAA2, 'M', u'甆'), + (0xFAA3, 'M', u'画'), + (0xFAA4, 'M', u'瘝'), + (0xFAA5, 'M', u'瘟'), + (0xFAA6, 'M', u'益'), + (0xFAA7, 'M', u'盛'), + (0xFAA8, 'M', u'直'), + (0xFAA9, 'M', u'睊'), + (0xFAAA, 'M', u'着'), + (0xFAAB, 'M', u'磌'), + (0xFAAC, 'M', u'窱'), + (0xFAAD, 'M', u'節'), + (0xFAAE, 'M', u'类'), + (0xFAAF, 'M', u'絛'), + (0xFAB0, 'M', u'練'), + (0xFAB1, 'M', u'缾'), + (0xFAB2, 'M', u'者'), + (0xFAB3, 'M', u'荒'), + (0xFAB4, 'M', u'華'), + (0xFAB5, 'M', u'蝹'), + (0xFAB6, 'M', u'襁'), + (0xFAB7, 'M', u'覆'), + (0xFAB8, 'M', u'視'), + (0xFAB9, 'M', u'調'), + (0xFABA, 'M', u'諸'), + (0xFABB, 'M', u'請'), + (0xFABC, 'M', u'謁'), + (0xFABD, 'M', u'諾'), + (0xFABE, 'M', u'諭'), + (0xFABF, 'M', u'謹'), + (0xFAC0, 'M', u'變'), + (0xFAC1, 'M', u'贈'), + (0xFAC2, 'M', u'輸'), + (0xFAC3, 'M', u'遲'), + (0xFAC4, 'M', u'醙'), + (0xFAC5, 'M', u'鉶'), + (0xFAC6, 'M', u'陼'), + (0xFAC7, 'M', u'難'), + (0xFAC8, 'M', u'靖'), + (0xFAC9, 'M', u'韛'), + (0xFACA, 'M', u'響'), + (0xFACB, 'M', u'頋'), + (0xFACC, 'M', u'頻'), + (0xFACD, 'M', u'鬒'), + (0xFACE, 'M', u'龜'), + (0xFACF, 'M', u'𢡊'), + (0xFAD0, 'M', u'𢡄'), + (0xFAD1, 'M', u'𣏕'), + (0xFAD2, 'M', u'㮝'), + (0xFAD3, 'M', u'䀘'), + (0xFAD4, 'M', u'䀹'), + (0xFAD5, 'M', u'𥉉'), + (0xFAD6, 'M', u'𥳐'), + (0xFAD7, 'M', u'𧻓'), + (0xFAD8, 'M', u'齃'), + (0xFAD9, 'M', u'龎'), + (0xFADA, 'X'), + (0xFB00, 'M', u'ff'), + (0xFB01, 'M', u'fi'), + (0xFB02, 'M', u'fl'), + (0xFB03, 'M', u'ffi'), + (0xFB04, 'M', u'ffl'), + (0xFB05, 'M', u'st'), + (0xFB07, 'X'), + (0xFB13, 'M', u'մն'), + (0xFB14, 'M', u'մե'), + (0xFB15, 'M', u'մի'), + (0xFB16, 'M', u'վն'), + ] + +def _seg_44(): + return [ + (0xFB17, 'M', u'մխ'), + (0xFB18, 'X'), + (0xFB1D, 'M', u'יִ'), + (0xFB1E, 'V'), + (0xFB1F, 'M', u'ײַ'), + (0xFB20, 'M', u'ע'), + (0xFB21, 'M', u'א'), + (0xFB22, 'M', u'ד'), + (0xFB23, 'M', u'ה'), + (0xFB24, 'M', u'כ'), + (0xFB25, 'M', u'ל'), + (0xFB26, 'M', u'ם'), + (0xFB27, 'M', u'ר'), + (0xFB28, 'M', u'ת'), + (0xFB29, '3', u'+'), + (0xFB2A, 'M', u'שׁ'), + (0xFB2B, 'M', u'שׂ'), + (0xFB2C, 'M', u'שּׁ'), + (0xFB2D, 'M', u'שּׂ'), + (0xFB2E, 'M', u'אַ'), + (0xFB2F, 'M', u'אָ'), + (0xFB30, 'M', u'אּ'), + (0xFB31, 'M', u'בּ'), + (0xFB32, 'M', u'גּ'), + (0xFB33, 'M', u'דּ'), + (0xFB34, 'M', u'הּ'), + (0xFB35, 'M', u'וּ'), + (0xFB36, 'M', u'זּ'), + (0xFB37, 'X'), + (0xFB38, 'M', u'טּ'), + (0xFB39, 'M', u'יּ'), + (0xFB3A, 'M', u'ךּ'), + (0xFB3B, 'M', u'כּ'), + (0xFB3C, 'M', u'לּ'), + (0xFB3D, 'X'), + (0xFB3E, 'M', u'מּ'), + (0xFB3F, 'X'), + (0xFB40, 'M', u'נּ'), + (0xFB41, 'M', u'סּ'), + (0xFB42, 'X'), + (0xFB43, 'M', u'ףּ'), + (0xFB44, 'M', u'פּ'), + (0xFB45, 'X'), + (0xFB46, 'M', u'צּ'), + (0xFB47, 'M', u'קּ'), + (0xFB48, 'M', u'רּ'), + (0xFB49, 'M', u'שּ'), + (0xFB4A, 'M', u'תּ'), + (0xFB4B, 'M', u'וֹ'), + (0xFB4C, 'M', u'בֿ'), + (0xFB4D, 'M', u'כֿ'), + (0xFB4E, 'M', u'פֿ'), + (0xFB4F, 'M', u'אל'), + (0xFB50, 'M', u'ٱ'), + (0xFB52, 'M', u'ٻ'), + (0xFB56, 'M', u'پ'), + (0xFB5A, 'M', u'ڀ'), + (0xFB5E, 'M', u'ٺ'), + (0xFB62, 'M', u'ٿ'), + (0xFB66, 'M', u'ٹ'), + (0xFB6A, 'M', u'ڤ'), + (0xFB6E, 'M', u'ڦ'), + (0xFB72, 'M', u'ڄ'), + (0xFB76, 'M', u'ڃ'), + (0xFB7A, 'M', u'چ'), + (0xFB7E, 'M', u'ڇ'), + (0xFB82, 'M', u'ڍ'), + (0xFB84, 'M', u'ڌ'), + (0xFB86, 'M', u'ڎ'), + (0xFB88, 'M', u'ڈ'), + (0xFB8A, 'M', u'ژ'), + (0xFB8C, 'M', u'ڑ'), + (0xFB8E, 'M', u'ک'), + (0xFB92, 'M', u'گ'), + (0xFB96, 'M', u'ڳ'), + (0xFB9A, 'M', u'ڱ'), + (0xFB9E, 'M', u'ں'), + (0xFBA0, 'M', u'ڻ'), + (0xFBA4, 'M', u'ۀ'), + (0xFBA6, 'M', u'ہ'), + (0xFBAA, 'M', u'ھ'), + (0xFBAE, 'M', u'ے'), + (0xFBB0, 'M', u'ۓ'), + (0xFBB2, 'V'), + (0xFBC2, 'X'), + (0xFBD3, 'M', u'ڭ'), + (0xFBD7, 'M', u'ۇ'), + (0xFBD9, 'M', u'ۆ'), + (0xFBDB, 'M', u'ۈ'), + (0xFBDD, 'M', u'ۇٴ'), + (0xFBDE, 'M', u'ۋ'), + (0xFBE0, 'M', u'ۅ'), + (0xFBE2, 'M', u'ۉ'), + (0xFBE4, 'M', u'ې'), + (0xFBE8, 'M', u'ى'), + (0xFBEA, 'M', u'ئا'), + (0xFBEC, 'M', u'ئە'), + (0xFBEE, 'M', u'ئو'), + (0xFBF0, 'M', u'ئۇ'), + (0xFBF2, 'M', u'ئۆ'), + ] + +def _seg_45(): + return [ + (0xFBF4, 'M', u'ئۈ'), + (0xFBF6, 'M', u'ئې'), + (0xFBF9, 'M', u'ئى'), + (0xFBFC, 'M', u'ی'), + (0xFC00, 'M', u'ئج'), + (0xFC01, 'M', u'ئح'), + (0xFC02, 'M', u'ئم'), + (0xFC03, 'M', u'ئى'), + (0xFC04, 'M', u'ئي'), + (0xFC05, 'M', u'بج'), + (0xFC06, 'M', u'بح'), + (0xFC07, 'M', u'بخ'), + (0xFC08, 'M', u'بم'), + (0xFC09, 'M', u'بى'), + (0xFC0A, 'M', u'بي'), + (0xFC0B, 'M', u'تج'), + (0xFC0C, 'M', u'تح'), + (0xFC0D, 'M', u'تخ'), + (0xFC0E, 'M', u'تم'), + (0xFC0F, 'M', u'تى'), + (0xFC10, 'M', u'تي'), + (0xFC11, 'M', u'ثج'), + (0xFC12, 'M', u'ثم'), + (0xFC13, 'M', u'ثى'), + (0xFC14, 'M', u'ثي'), + (0xFC15, 'M', u'جح'), + (0xFC16, 'M', u'جم'), + (0xFC17, 'M', u'حج'), + (0xFC18, 'M', u'حم'), + (0xFC19, 'M', u'خج'), + (0xFC1A, 'M', u'خح'), + (0xFC1B, 'M', u'خم'), + (0xFC1C, 'M', u'سج'), + (0xFC1D, 'M', u'سح'), + (0xFC1E, 'M', u'سخ'), + (0xFC1F, 'M', u'سم'), + (0xFC20, 'M', u'صح'), + (0xFC21, 'M', u'صم'), + (0xFC22, 'M', u'ضج'), + (0xFC23, 'M', u'ضح'), + (0xFC24, 'M', u'ضخ'), + (0xFC25, 'M', u'ضم'), + (0xFC26, 'M', u'طح'), + (0xFC27, 'M', u'طم'), + (0xFC28, 'M', u'ظم'), + (0xFC29, 'M', u'عج'), + (0xFC2A, 'M', u'عم'), + (0xFC2B, 'M', u'غج'), + (0xFC2C, 'M', u'غم'), + (0xFC2D, 'M', u'فج'), + (0xFC2E, 'M', u'فح'), + (0xFC2F, 'M', u'فخ'), + (0xFC30, 'M', u'فم'), + (0xFC31, 'M', u'فى'), + (0xFC32, 'M', u'في'), + (0xFC33, 'M', u'قح'), + (0xFC34, 'M', u'قم'), + (0xFC35, 'M', u'قى'), + (0xFC36, 'M', u'قي'), + (0xFC37, 'M', u'كا'), + (0xFC38, 'M', u'كج'), + (0xFC39, 'M', u'كح'), + (0xFC3A, 'M', u'كخ'), + (0xFC3B, 'M', u'كل'), + (0xFC3C, 'M', u'كم'), + (0xFC3D, 'M', u'كى'), + (0xFC3E, 'M', u'كي'), + (0xFC3F, 'M', u'لج'), + (0xFC40, 'M', u'لح'), + (0xFC41, 'M', u'لخ'), + (0xFC42, 'M', u'لم'), + (0xFC43, 'M', u'لى'), + (0xFC44, 'M', u'لي'), + (0xFC45, 'M', u'مج'), + (0xFC46, 'M', u'مح'), + (0xFC47, 'M', u'مخ'), + (0xFC48, 'M', u'مم'), + (0xFC49, 'M', u'مى'), + (0xFC4A, 'M', u'مي'), + (0xFC4B, 'M', u'نج'), + (0xFC4C, 'M', u'نح'), + (0xFC4D, 'M', u'نخ'), + (0xFC4E, 'M', u'نم'), + (0xFC4F, 'M', u'نى'), + (0xFC50, 'M', u'ني'), + (0xFC51, 'M', u'هج'), + (0xFC52, 'M', u'هم'), + (0xFC53, 'M', u'هى'), + (0xFC54, 'M', u'هي'), + (0xFC55, 'M', u'يج'), + (0xFC56, 'M', u'يح'), + (0xFC57, 'M', u'يخ'), + (0xFC58, 'M', u'يم'), + (0xFC59, 'M', u'يى'), + (0xFC5A, 'M', u'يي'), + (0xFC5B, 'M', u'ذٰ'), + (0xFC5C, 'M', u'رٰ'), + (0xFC5D, 'M', u'ىٰ'), + (0xFC5E, '3', u' ٌّ'), + (0xFC5F, '3', u' ٍّ'), + ] + +def _seg_46(): + return [ + (0xFC60, '3', u' َّ'), + (0xFC61, '3', u' ُّ'), + (0xFC62, '3', u' ِّ'), + (0xFC63, '3', u' ّٰ'), + (0xFC64, 'M', u'ئر'), + (0xFC65, 'M', u'ئز'), + (0xFC66, 'M', u'ئم'), + (0xFC67, 'M', u'ئن'), + (0xFC68, 'M', u'ئى'), + (0xFC69, 'M', u'ئي'), + (0xFC6A, 'M', u'بر'), + (0xFC6B, 'M', u'بز'), + (0xFC6C, 'M', u'بم'), + (0xFC6D, 'M', u'بن'), + (0xFC6E, 'M', u'بى'), + (0xFC6F, 'M', u'بي'), + (0xFC70, 'M', u'تر'), + (0xFC71, 'M', u'تز'), + (0xFC72, 'M', u'تم'), + (0xFC73, 'M', u'تن'), + (0xFC74, 'M', u'تى'), + (0xFC75, 'M', u'تي'), + (0xFC76, 'M', u'ثر'), + (0xFC77, 'M', u'ثز'), + (0xFC78, 'M', u'ثم'), + (0xFC79, 'M', u'ثن'), + (0xFC7A, 'M', u'ثى'), + (0xFC7B, 'M', u'ثي'), + (0xFC7C, 'M', u'فى'), + (0xFC7D, 'M', u'في'), + (0xFC7E, 'M', u'قى'), + (0xFC7F, 'M', u'قي'), + (0xFC80, 'M', u'كا'), + (0xFC81, 'M', u'كل'), + (0xFC82, 'M', u'كم'), + (0xFC83, 'M', u'كى'), + (0xFC84, 'M', u'كي'), + (0xFC85, 'M', u'لم'), + (0xFC86, 'M', u'لى'), + (0xFC87, 'M', u'لي'), + (0xFC88, 'M', u'ما'), + (0xFC89, 'M', u'مم'), + (0xFC8A, 'M', u'نر'), + (0xFC8B, 'M', u'نز'), + (0xFC8C, 'M', u'نم'), + (0xFC8D, 'M', u'نن'), + (0xFC8E, 'M', u'نى'), + (0xFC8F, 'M', u'ني'), + (0xFC90, 'M', u'ىٰ'), + (0xFC91, 'M', u'ير'), + (0xFC92, 'M', u'يز'), + (0xFC93, 'M', u'يم'), + (0xFC94, 'M', u'ين'), + (0xFC95, 'M', u'يى'), + (0xFC96, 'M', u'يي'), + (0xFC97, 'M', u'ئج'), + (0xFC98, 'M', u'ئح'), + (0xFC99, 'M', u'ئخ'), + (0xFC9A, 'M', u'ئم'), + (0xFC9B, 'M', u'ئه'), + (0xFC9C, 'M', u'بج'), + (0xFC9D, 'M', u'بح'), + (0xFC9E, 'M', u'بخ'), + (0xFC9F, 'M', u'بم'), + (0xFCA0, 'M', u'به'), + (0xFCA1, 'M', u'تج'), + (0xFCA2, 'M', u'تح'), + (0xFCA3, 'M', u'تخ'), + (0xFCA4, 'M', u'تم'), + (0xFCA5, 'M', u'ته'), + (0xFCA6, 'M', u'ثم'), + (0xFCA7, 'M', u'جح'), + (0xFCA8, 'M', u'جم'), + (0xFCA9, 'M', u'حج'), + (0xFCAA, 'M', u'حم'), + (0xFCAB, 'M', u'خج'), + (0xFCAC, 'M', u'خم'), + (0xFCAD, 'M', u'سج'), + (0xFCAE, 'M', u'سح'), + (0xFCAF, 'M', u'سخ'), + (0xFCB0, 'M', u'سم'), + (0xFCB1, 'M', u'صح'), + (0xFCB2, 'M', u'صخ'), + (0xFCB3, 'M', u'صم'), + (0xFCB4, 'M', u'ضج'), + (0xFCB5, 'M', u'ضح'), + (0xFCB6, 'M', u'ضخ'), + (0xFCB7, 'M', u'ضم'), + (0xFCB8, 'M', u'طح'), + (0xFCB9, 'M', u'ظم'), + (0xFCBA, 'M', u'عج'), + (0xFCBB, 'M', u'عم'), + (0xFCBC, 'M', u'غج'), + (0xFCBD, 'M', u'غم'), + (0xFCBE, 'M', u'فج'), + (0xFCBF, 'M', u'فح'), + (0xFCC0, 'M', u'فخ'), + (0xFCC1, 'M', u'فم'), + (0xFCC2, 'M', u'قح'), + (0xFCC3, 'M', u'قم'), + ] + +def _seg_47(): + return [ + (0xFCC4, 'M', u'كج'), + (0xFCC5, 'M', u'كح'), + (0xFCC6, 'M', u'كخ'), + (0xFCC7, 'M', u'كل'), + (0xFCC8, 'M', u'كم'), + (0xFCC9, 'M', u'لج'), + (0xFCCA, 'M', u'لح'), + (0xFCCB, 'M', u'لخ'), + (0xFCCC, 'M', u'لم'), + (0xFCCD, 'M', u'له'), + (0xFCCE, 'M', u'مج'), + (0xFCCF, 'M', u'مح'), + (0xFCD0, 'M', u'مخ'), + (0xFCD1, 'M', u'مم'), + (0xFCD2, 'M', u'نج'), + (0xFCD3, 'M', u'نح'), + (0xFCD4, 'M', u'نخ'), + (0xFCD5, 'M', u'نم'), + (0xFCD6, 'M', u'نه'), + (0xFCD7, 'M', u'هج'), + (0xFCD8, 'M', u'هم'), + (0xFCD9, 'M', u'هٰ'), + (0xFCDA, 'M', u'يج'), + (0xFCDB, 'M', u'يح'), + (0xFCDC, 'M', u'يخ'), + (0xFCDD, 'M', u'يم'), + (0xFCDE, 'M', u'يه'), + (0xFCDF, 'M', u'ئم'), + (0xFCE0, 'M', u'ئه'), + (0xFCE1, 'M', u'بم'), + (0xFCE2, 'M', u'به'), + (0xFCE3, 'M', u'تم'), + (0xFCE4, 'M', u'ته'), + (0xFCE5, 'M', u'ثم'), + (0xFCE6, 'M', u'ثه'), + (0xFCE7, 'M', u'سم'), + (0xFCE8, 'M', u'سه'), + (0xFCE9, 'M', u'شم'), + (0xFCEA, 'M', u'شه'), + (0xFCEB, 'M', u'كل'), + (0xFCEC, 'M', u'كم'), + (0xFCED, 'M', u'لم'), + (0xFCEE, 'M', u'نم'), + (0xFCEF, 'M', u'نه'), + (0xFCF0, 'M', u'يم'), + (0xFCF1, 'M', u'يه'), + (0xFCF2, 'M', u'ـَّ'), + (0xFCF3, 'M', u'ـُّ'), + (0xFCF4, 'M', u'ـِّ'), + (0xFCF5, 'M', u'طى'), + (0xFCF6, 'M', u'طي'), + (0xFCF7, 'M', u'عى'), + (0xFCF8, 'M', u'عي'), + (0xFCF9, 'M', u'غى'), + (0xFCFA, 'M', u'غي'), + (0xFCFB, 'M', u'سى'), + (0xFCFC, 'M', u'سي'), + (0xFCFD, 'M', u'شى'), + (0xFCFE, 'M', u'شي'), + (0xFCFF, 'M', u'حى'), + (0xFD00, 'M', u'حي'), + (0xFD01, 'M', u'جى'), + (0xFD02, 'M', u'جي'), + (0xFD03, 'M', u'خى'), + (0xFD04, 'M', u'خي'), + (0xFD05, 'M', u'صى'), + (0xFD06, 'M', u'صي'), + (0xFD07, 'M', u'ضى'), + (0xFD08, 'M', u'ضي'), + (0xFD09, 'M', u'شج'), + (0xFD0A, 'M', u'شح'), + (0xFD0B, 'M', u'شخ'), + (0xFD0C, 'M', u'شم'), + (0xFD0D, 'M', u'شر'), + (0xFD0E, 'M', u'سر'), + (0xFD0F, 'M', u'صر'), + (0xFD10, 'M', u'ضر'), + (0xFD11, 'M', u'طى'), + (0xFD12, 'M', u'طي'), + (0xFD13, 'M', u'عى'), + (0xFD14, 'M', u'عي'), + (0xFD15, 'M', u'غى'), + (0xFD16, 'M', u'غي'), + (0xFD17, 'M', u'سى'), + (0xFD18, 'M', u'سي'), + (0xFD19, 'M', u'شى'), + (0xFD1A, 'M', u'شي'), + (0xFD1B, 'M', u'حى'), + (0xFD1C, 'M', u'حي'), + (0xFD1D, 'M', u'جى'), + (0xFD1E, 'M', u'جي'), + (0xFD1F, 'M', u'خى'), + (0xFD20, 'M', u'خي'), + (0xFD21, 'M', u'صى'), + (0xFD22, 'M', u'صي'), + (0xFD23, 'M', u'ضى'), + (0xFD24, 'M', u'ضي'), + (0xFD25, 'M', u'شج'), + (0xFD26, 'M', u'شح'), + (0xFD27, 'M', u'شخ'), + ] + +def _seg_48(): + return [ + (0xFD28, 'M', u'شم'), + (0xFD29, 'M', u'شر'), + (0xFD2A, 'M', u'سر'), + (0xFD2B, 'M', u'صر'), + (0xFD2C, 'M', u'ضر'), + (0xFD2D, 'M', u'شج'), + (0xFD2E, 'M', u'شح'), + (0xFD2F, 'M', u'شخ'), + (0xFD30, 'M', u'شم'), + (0xFD31, 'M', u'سه'), + (0xFD32, 'M', u'شه'), + (0xFD33, 'M', u'طم'), + (0xFD34, 'M', u'سج'), + (0xFD35, 'M', u'سح'), + (0xFD36, 'M', u'سخ'), + (0xFD37, 'M', u'شج'), + (0xFD38, 'M', u'شح'), + (0xFD39, 'M', u'شخ'), + (0xFD3A, 'M', u'طم'), + (0xFD3B, 'M', u'ظم'), + (0xFD3C, 'M', u'اً'), + (0xFD3E, 'V'), + (0xFD40, 'X'), + (0xFD50, 'M', u'تجم'), + (0xFD51, 'M', u'تحج'), + (0xFD53, 'M', u'تحم'), + (0xFD54, 'M', u'تخم'), + (0xFD55, 'M', u'تمج'), + (0xFD56, 'M', u'تمح'), + (0xFD57, 'M', u'تمخ'), + (0xFD58, 'M', u'جمح'), + (0xFD5A, 'M', u'حمي'), + (0xFD5B, 'M', u'حمى'), + (0xFD5C, 'M', u'سحج'), + (0xFD5D, 'M', u'سجح'), + (0xFD5E, 'M', u'سجى'), + (0xFD5F, 'M', u'سمح'), + (0xFD61, 'M', u'سمج'), + (0xFD62, 'M', u'سمم'), + (0xFD64, 'M', u'صحح'), + (0xFD66, 'M', u'صمم'), + (0xFD67, 'M', u'شحم'), + (0xFD69, 'M', u'شجي'), + (0xFD6A, 'M', u'شمخ'), + (0xFD6C, 'M', u'شمم'), + (0xFD6E, 'M', u'ضحى'), + (0xFD6F, 'M', u'ضخم'), + (0xFD71, 'M', u'طمح'), + (0xFD73, 'M', u'طمم'), + (0xFD74, 'M', u'طمي'), + (0xFD75, 'M', u'عجم'), + (0xFD76, 'M', u'عمم'), + (0xFD78, 'M', u'عمى'), + (0xFD79, 'M', u'غمم'), + (0xFD7A, 'M', u'غمي'), + (0xFD7B, 'M', u'غمى'), + (0xFD7C, 'M', u'فخم'), + (0xFD7E, 'M', u'قمح'), + (0xFD7F, 'M', u'قمم'), + (0xFD80, 'M', u'لحم'), + (0xFD81, 'M', u'لحي'), + (0xFD82, 'M', u'لحى'), + (0xFD83, 'M', u'لجج'), + (0xFD85, 'M', u'لخم'), + (0xFD87, 'M', u'لمح'), + (0xFD89, 'M', u'محج'), + (0xFD8A, 'M', u'محم'), + (0xFD8B, 'M', u'محي'), + (0xFD8C, 'M', u'مجح'), + (0xFD8D, 'M', u'مجم'), + (0xFD8E, 'M', u'مخج'), + (0xFD8F, 'M', u'مخم'), + (0xFD90, 'X'), + (0xFD92, 'M', u'مجخ'), + (0xFD93, 'M', u'همج'), + (0xFD94, 'M', u'همم'), + (0xFD95, 'M', u'نحم'), + (0xFD96, 'M', u'نحى'), + (0xFD97, 'M', u'نجم'), + (0xFD99, 'M', u'نجى'), + (0xFD9A, 'M', u'نمي'), + (0xFD9B, 'M', u'نمى'), + (0xFD9C, 'M', u'يمم'), + (0xFD9E, 'M', u'بخي'), + (0xFD9F, 'M', u'تجي'), + (0xFDA0, 'M', u'تجى'), + (0xFDA1, 'M', u'تخي'), + (0xFDA2, 'M', u'تخى'), + (0xFDA3, 'M', u'تمي'), + (0xFDA4, 'M', u'تمى'), + (0xFDA5, 'M', u'جمي'), + (0xFDA6, 'M', u'جحى'), + (0xFDA7, 'M', u'جمى'), + (0xFDA8, 'M', u'سخى'), + (0xFDA9, 'M', u'صحي'), + (0xFDAA, 'M', u'شحي'), + (0xFDAB, 'M', u'ضحي'), + (0xFDAC, 'M', u'لجي'), + (0xFDAD, 'M', u'لمي'), + (0xFDAE, 'M', u'يحي'), + ] + +def _seg_49(): + return [ + (0xFDAF, 'M', u'يجي'), + (0xFDB0, 'M', u'يمي'), + (0xFDB1, 'M', u'ممي'), + (0xFDB2, 'M', u'قمي'), + (0xFDB3, 'M', u'نحي'), + (0xFDB4, 'M', u'قمح'), + (0xFDB5, 'M', u'لحم'), + (0xFDB6, 'M', u'عمي'), + (0xFDB7, 'M', u'كمي'), + (0xFDB8, 'M', u'نجح'), + (0xFDB9, 'M', u'مخي'), + (0xFDBA, 'M', u'لجم'), + (0xFDBB, 'M', u'كمم'), + (0xFDBC, 'M', u'لجم'), + (0xFDBD, 'M', u'نجح'), + (0xFDBE, 'M', u'جحي'), + (0xFDBF, 'M', u'حجي'), + (0xFDC0, 'M', u'مجي'), + (0xFDC1, 'M', u'فمي'), + (0xFDC2, 'M', u'بحي'), + (0xFDC3, 'M', u'كمم'), + (0xFDC4, 'M', u'عجم'), + (0xFDC5, 'M', u'صمم'), + (0xFDC6, 'M', u'سخي'), + (0xFDC7, 'M', u'نجي'), + (0xFDC8, 'X'), + (0xFDF0, 'M', u'صلے'), + (0xFDF1, 'M', u'قلے'), + (0xFDF2, 'M', u'الله'), + (0xFDF3, 'M', u'اكبر'), + (0xFDF4, 'M', u'محمد'), + (0xFDF5, 'M', u'صلعم'), + (0xFDF6, 'M', u'رسول'), + (0xFDF7, 'M', u'عليه'), + (0xFDF8, 'M', u'وسلم'), + (0xFDF9, 'M', u'صلى'), + (0xFDFA, '3', u'صلى الله عليه وسلم'), + (0xFDFB, '3', u'جل جلاله'), + (0xFDFC, 'M', u'ریال'), + (0xFDFD, 'V'), + (0xFDFE, 'X'), + (0xFE00, 'I'), + (0xFE10, '3', u','), + (0xFE11, 'M', u'、'), + (0xFE12, 'X'), + (0xFE13, '3', u':'), + (0xFE14, '3', u';'), + (0xFE15, '3', u'!'), + (0xFE16, '3', u'?'), + (0xFE17, 'M', u'〖'), + (0xFE18, 'M', u'〗'), + (0xFE19, 'X'), + (0xFE20, 'V'), + (0xFE30, 'X'), + (0xFE31, 'M', u'—'), + (0xFE32, 'M', u'–'), + (0xFE33, '3', u'_'), + (0xFE35, '3', u'('), + (0xFE36, '3', u')'), + (0xFE37, '3', u'{'), + (0xFE38, '3', u'}'), + (0xFE39, 'M', u'〔'), + (0xFE3A, 'M', u'〕'), + (0xFE3B, 'M', u'【'), + (0xFE3C, 'M', u'】'), + (0xFE3D, 'M', u'《'), + (0xFE3E, 'M', u'》'), + (0xFE3F, 'M', u'〈'), + (0xFE40, 'M', u'〉'), + (0xFE41, 'M', u'「'), + (0xFE42, 'M', u'」'), + (0xFE43, 'M', u'『'), + (0xFE44, 'M', u'』'), + (0xFE45, 'V'), + (0xFE47, '3', u'['), + (0xFE48, '3', u']'), + (0xFE49, '3', u' ̅'), + (0xFE4D, '3', u'_'), + (0xFE50, '3', u','), + (0xFE51, 'M', u'、'), + (0xFE52, 'X'), + (0xFE54, '3', u';'), + (0xFE55, '3', u':'), + (0xFE56, '3', u'?'), + (0xFE57, '3', u'!'), + (0xFE58, 'M', u'—'), + (0xFE59, '3', u'('), + (0xFE5A, '3', u')'), + (0xFE5B, '3', u'{'), + (0xFE5C, '3', u'}'), + (0xFE5D, 'M', u'〔'), + (0xFE5E, 'M', u'〕'), + (0xFE5F, '3', u'#'), + (0xFE60, '3', u'&'), + (0xFE61, '3', u'*'), + (0xFE62, '3', u'+'), + (0xFE63, 'M', u'-'), + (0xFE64, '3', u'<'), + (0xFE65, '3', u'>'), + (0xFE66, '3', u'='), + ] + +def _seg_50(): + return [ + (0xFE67, 'X'), + (0xFE68, '3', u'\\'), + (0xFE69, '3', u'$'), + (0xFE6A, '3', u'%'), + (0xFE6B, '3', u'@'), + (0xFE6C, 'X'), + (0xFE70, '3', u' ً'), + (0xFE71, 'M', u'ـً'), + (0xFE72, '3', u' ٌ'), + (0xFE73, 'V'), + (0xFE74, '3', u' ٍ'), + (0xFE75, 'X'), + (0xFE76, '3', u' َ'), + (0xFE77, 'M', u'ـَ'), + (0xFE78, '3', u' ُ'), + (0xFE79, 'M', u'ـُ'), + (0xFE7A, '3', u' ِ'), + (0xFE7B, 'M', u'ـِ'), + (0xFE7C, '3', u' ّ'), + (0xFE7D, 'M', u'ـّ'), + (0xFE7E, '3', u' ْ'), + (0xFE7F, 'M', u'ـْ'), + (0xFE80, 'M', u'ء'), + (0xFE81, 'M', u'آ'), + (0xFE83, 'M', u'أ'), + (0xFE85, 'M', u'ؤ'), + (0xFE87, 'M', u'إ'), + (0xFE89, 'M', u'ئ'), + (0xFE8D, 'M', u'ا'), + (0xFE8F, 'M', u'ب'), + (0xFE93, 'M', u'ة'), + (0xFE95, 'M', u'ت'), + (0xFE99, 'M', u'ث'), + (0xFE9D, 'M', u'ج'), + (0xFEA1, 'M', u'ح'), + (0xFEA5, 'M', u'خ'), + (0xFEA9, 'M', u'د'), + (0xFEAB, 'M', u'ذ'), + (0xFEAD, 'M', u'ر'), + (0xFEAF, 'M', u'ز'), + (0xFEB1, 'M', u'س'), + (0xFEB5, 'M', u'ش'), + (0xFEB9, 'M', u'ص'), + (0xFEBD, 'M', u'ض'), + (0xFEC1, 'M', u'ط'), + (0xFEC5, 'M', u'ظ'), + (0xFEC9, 'M', u'ع'), + (0xFECD, 'M', u'غ'), + (0xFED1, 'M', u'ف'), + (0xFED5, 'M', u'ق'), + (0xFED9, 'M', u'ك'), + (0xFEDD, 'M', u'ل'), + (0xFEE1, 'M', u'م'), + (0xFEE5, 'M', u'ن'), + (0xFEE9, 'M', u'ه'), + (0xFEED, 'M', u'و'), + (0xFEEF, 'M', u'ى'), + (0xFEF1, 'M', u'ي'), + (0xFEF5, 'M', u'لآ'), + (0xFEF7, 'M', u'لأ'), + (0xFEF9, 'M', u'لإ'), + (0xFEFB, 'M', u'لا'), + (0xFEFD, 'X'), + (0xFEFF, 'I'), + (0xFF00, 'X'), + (0xFF01, '3', u'!'), + (0xFF02, '3', u'"'), + (0xFF03, '3', u'#'), + (0xFF04, '3', u'$'), + (0xFF05, '3', u'%'), + (0xFF06, '3', u'&'), + (0xFF07, '3', u'\''), + (0xFF08, '3', u'('), + (0xFF09, '3', u')'), + (0xFF0A, '3', u'*'), + (0xFF0B, '3', u'+'), + (0xFF0C, '3', u','), + (0xFF0D, 'M', u'-'), + (0xFF0E, 'M', u'.'), + (0xFF0F, '3', u'/'), + (0xFF10, 'M', u'0'), + (0xFF11, 'M', u'1'), + (0xFF12, 'M', u'2'), + (0xFF13, 'M', u'3'), + (0xFF14, 'M', u'4'), + (0xFF15, 'M', u'5'), + (0xFF16, 'M', u'6'), + (0xFF17, 'M', u'7'), + (0xFF18, 'M', u'8'), + (0xFF19, 'M', u'9'), + (0xFF1A, '3', u':'), + (0xFF1B, '3', u';'), + (0xFF1C, '3', u'<'), + (0xFF1D, '3', u'='), + (0xFF1E, '3', u'>'), + (0xFF1F, '3', u'?'), + (0xFF20, '3', u'@'), + (0xFF21, 'M', u'a'), + (0xFF22, 'M', u'b'), + (0xFF23, 'M', u'c'), + ] + +def _seg_51(): + return [ + (0xFF24, 'M', u'd'), + (0xFF25, 'M', u'e'), + (0xFF26, 'M', u'f'), + (0xFF27, 'M', u'g'), + (0xFF28, 'M', u'h'), + (0xFF29, 'M', u'i'), + (0xFF2A, 'M', u'j'), + (0xFF2B, 'M', u'k'), + (0xFF2C, 'M', u'l'), + (0xFF2D, 'M', u'm'), + (0xFF2E, 'M', u'n'), + (0xFF2F, 'M', u'o'), + (0xFF30, 'M', u'p'), + (0xFF31, 'M', u'q'), + (0xFF32, 'M', u'r'), + (0xFF33, 'M', u's'), + (0xFF34, 'M', u't'), + (0xFF35, 'M', u'u'), + (0xFF36, 'M', u'v'), + (0xFF37, 'M', u'w'), + (0xFF38, 'M', u'x'), + (0xFF39, 'M', u'y'), + (0xFF3A, 'M', u'z'), + (0xFF3B, '3', u'['), + (0xFF3C, '3', u'\\'), + (0xFF3D, '3', u']'), + (0xFF3E, '3', u'^'), + (0xFF3F, '3', u'_'), + (0xFF40, '3', u'`'), + (0xFF41, 'M', u'a'), + (0xFF42, 'M', u'b'), + (0xFF43, 'M', u'c'), + (0xFF44, 'M', u'd'), + (0xFF45, 'M', u'e'), + (0xFF46, 'M', u'f'), + (0xFF47, 'M', u'g'), + (0xFF48, 'M', u'h'), + (0xFF49, 'M', u'i'), + (0xFF4A, 'M', u'j'), + (0xFF4B, 'M', u'k'), + (0xFF4C, 'M', u'l'), + (0xFF4D, 'M', u'm'), + (0xFF4E, 'M', u'n'), + (0xFF4F, 'M', u'o'), + (0xFF50, 'M', u'p'), + (0xFF51, 'M', u'q'), + (0xFF52, 'M', u'r'), + (0xFF53, 'M', u's'), + (0xFF54, 'M', u't'), + (0xFF55, 'M', u'u'), + (0xFF56, 'M', u'v'), + (0xFF57, 'M', u'w'), + (0xFF58, 'M', u'x'), + (0xFF59, 'M', u'y'), + (0xFF5A, 'M', u'z'), + (0xFF5B, '3', u'{'), + (0xFF5C, '3', u'|'), + (0xFF5D, '3', u'}'), + (0xFF5E, '3', u'~'), + (0xFF5F, 'M', u'⦅'), + (0xFF60, 'M', u'⦆'), + (0xFF61, 'M', u'.'), + (0xFF62, 'M', u'「'), + (0xFF63, 'M', u'」'), + (0xFF64, 'M', u'、'), + (0xFF65, 'M', u'・'), + (0xFF66, 'M', u'ヲ'), + (0xFF67, 'M', u'ァ'), + (0xFF68, 'M', u'ィ'), + (0xFF69, 'M', u'ゥ'), + (0xFF6A, 'M', u'ェ'), + (0xFF6B, 'M', u'ォ'), + (0xFF6C, 'M', u'ャ'), + (0xFF6D, 'M', u'ュ'), + (0xFF6E, 'M', u'ョ'), + (0xFF6F, 'M', u'ッ'), + (0xFF70, 'M', u'ー'), + (0xFF71, 'M', u'ア'), + (0xFF72, 'M', u'イ'), + (0xFF73, 'M', u'ウ'), + (0xFF74, 'M', u'エ'), + (0xFF75, 'M', u'オ'), + (0xFF76, 'M', u'カ'), + (0xFF77, 'M', u'キ'), + (0xFF78, 'M', u'ク'), + (0xFF79, 'M', u'ケ'), + (0xFF7A, 'M', u'コ'), + (0xFF7B, 'M', u'サ'), + (0xFF7C, 'M', u'シ'), + (0xFF7D, 'M', u'ス'), + (0xFF7E, 'M', u'セ'), + (0xFF7F, 'M', u'ソ'), + (0xFF80, 'M', u'タ'), + (0xFF81, 'M', u'チ'), + (0xFF82, 'M', u'ツ'), + (0xFF83, 'M', u'テ'), + (0xFF84, 'M', u'ト'), + (0xFF85, 'M', u'ナ'), + (0xFF86, 'M', u'ニ'), + (0xFF87, 'M', u'ヌ'), + ] + +def _seg_52(): + return [ + (0xFF88, 'M', u'ネ'), + (0xFF89, 'M', u'ノ'), + (0xFF8A, 'M', u'ハ'), + (0xFF8B, 'M', u'ヒ'), + (0xFF8C, 'M', u'フ'), + (0xFF8D, 'M', u'ヘ'), + (0xFF8E, 'M', u'ホ'), + (0xFF8F, 'M', u'マ'), + (0xFF90, 'M', u'ミ'), + (0xFF91, 'M', u'ム'), + (0xFF92, 'M', u'メ'), + (0xFF93, 'M', u'モ'), + (0xFF94, 'M', u'ヤ'), + (0xFF95, 'M', u'ユ'), + (0xFF96, 'M', u'ヨ'), + (0xFF97, 'M', u'ラ'), + (0xFF98, 'M', u'リ'), + (0xFF99, 'M', u'ル'), + (0xFF9A, 'M', u'レ'), + (0xFF9B, 'M', u'ロ'), + (0xFF9C, 'M', u'ワ'), + (0xFF9D, 'M', u'ン'), + (0xFF9E, 'M', u'゙'), + (0xFF9F, 'M', u'゚'), + (0xFFA0, 'X'), + (0xFFA1, 'M', u'ᄀ'), + (0xFFA2, 'M', u'ᄁ'), + (0xFFA3, 'M', u'ᆪ'), + (0xFFA4, 'M', u'ᄂ'), + (0xFFA5, 'M', u'ᆬ'), + (0xFFA6, 'M', u'ᆭ'), + (0xFFA7, 'M', u'ᄃ'), + (0xFFA8, 'M', u'ᄄ'), + (0xFFA9, 'M', u'ᄅ'), + (0xFFAA, 'M', u'ᆰ'), + (0xFFAB, 'M', u'ᆱ'), + (0xFFAC, 'M', u'ᆲ'), + (0xFFAD, 'M', u'ᆳ'), + (0xFFAE, 'M', u'ᆴ'), + (0xFFAF, 'M', u'ᆵ'), + (0xFFB0, 'M', u'ᄚ'), + (0xFFB1, 'M', u'ᄆ'), + (0xFFB2, 'M', u'ᄇ'), + (0xFFB3, 'M', u'ᄈ'), + (0xFFB4, 'M', u'ᄡ'), + (0xFFB5, 'M', u'ᄉ'), + (0xFFB6, 'M', u'ᄊ'), + (0xFFB7, 'M', u'ᄋ'), + (0xFFB8, 'M', u'ᄌ'), + (0xFFB9, 'M', u'ᄍ'), + (0xFFBA, 'M', u'ᄎ'), + (0xFFBB, 'M', u'ᄏ'), + (0xFFBC, 'M', u'ᄐ'), + (0xFFBD, 'M', u'ᄑ'), + (0xFFBE, 'M', u'ᄒ'), + (0xFFBF, 'X'), + (0xFFC2, 'M', u'ᅡ'), + (0xFFC3, 'M', u'ᅢ'), + (0xFFC4, 'M', u'ᅣ'), + (0xFFC5, 'M', u'ᅤ'), + (0xFFC6, 'M', u'ᅥ'), + (0xFFC7, 'M', u'ᅦ'), + (0xFFC8, 'X'), + (0xFFCA, 'M', u'ᅧ'), + (0xFFCB, 'M', u'ᅨ'), + (0xFFCC, 'M', u'ᅩ'), + (0xFFCD, 'M', u'ᅪ'), + (0xFFCE, 'M', u'ᅫ'), + (0xFFCF, 'M', u'ᅬ'), + (0xFFD0, 'X'), + (0xFFD2, 'M', u'ᅭ'), + (0xFFD3, 'M', u'ᅮ'), + (0xFFD4, 'M', u'ᅯ'), + (0xFFD5, 'M', u'ᅰ'), + (0xFFD6, 'M', u'ᅱ'), + (0xFFD7, 'M', u'ᅲ'), + (0xFFD8, 'X'), + (0xFFDA, 'M', u'ᅳ'), + (0xFFDB, 'M', u'ᅴ'), + (0xFFDC, 'M', u'ᅵ'), + (0xFFDD, 'X'), + (0xFFE0, 'M', u'¢'), + (0xFFE1, 'M', u'£'), + (0xFFE2, 'M', u'¬'), + (0xFFE3, '3', u' ̄'), + (0xFFE4, 'M', u'¦'), + (0xFFE5, 'M', u'¥'), + (0xFFE6, 'M', u'₩'), + (0xFFE7, 'X'), + (0xFFE8, 'M', u'│'), + (0xFFE9, 'M', u'←'), + (0xFFEA, 'M', u'↑'), + (0xFFEB, 'M', u'→'), + (0xFFEC, 'M', u'↓'), + (0xFFED, 'M', u'■'), + (0xFFEE, 'M', u'○'), + (0xFFEF, 'X'), + (0x10000, 'V'), + (0x1000C, 'X'), + (0x1000D, 'V'), + ] + +def _seg_53(): + return [ + (0x10027, 'X'), + (0x10028, 'V'), + (0x1003B, 'X'), + (0x1003C, 'V'), + (0x1003E, 'X'), + (0x1003F, 'V'), + (0x1004E, 'X'), + (0x10050, 'V'), + (0x1005E, 'X'), + (0x10080, 'V'), + (0x100FB, 'X'), + (0x10100, 'V'), + (0x10103, 'X'), + (0x10107, 'V'), + (0x10134, 'X'), + (0x10137, 'V'), + (0x1018F, 'X'), + (0x10190, 'V'), + (0x1019D, 'X'), + (0x101A0, 'V'), + (0x101A1, 'X'), + (0x101D0, 'V'), + (0x101FE, 'X'), + (0x10280, 'V'), + (0x1029D, 'X'), + (0x102A0, 'V'), + (0x102D1, 'X'), + (0x102E0, 'V'), + (0x102FC, 'X'), + (0x10300, 'V'), + (0x10324, 'X'), + (0x1032D, 'V'), + (0x1034B, 'X'), + (0x10350, 'V'), + (0x1037B, 'X'), + (0x10380, 'V'), + (0x1039E, 'X'), + (0x1039F, 'V'), + (0x103C4, 'X'), + (0x103C8, 'V'), + (0x103D6, 'X'), + (0x10400, 'M', u'𐐨'), + (0x10401, 'M', u'𐐩'), + (0x10402, 'M', u'𐐪'), + (0x10403, 'M', u'𐐫'), + (0x10404, 'M', u'𐐬'), + (0x10405, 'M', u'𐐭'), + (0x10406, 'M', u'𐐮'), + (0x10407, 'M', u'𐐯'), + (0x10408, 'M', u'𐐰'), + (0x10409, 'M', u'𐐱'), + (0x1040A, 'M', u'𐐲'), + (0x1040B, 'M', u'𐐳'), + (0x1040C, 'M', u'𐐴'), + (0x1040D, 'M', u'𐐵'), + (0x1040E, 'M', u'𐐶'), + (0x1040F, 'M', u'𐐷'), + (0x10410, 'M', u'𐐸'), + (0x10411, 'M', u'𐐹'), + (0x10412, 'M', u'𐐺'), + (0x10413, 'M', u'𐐻'), + (0x10414, 'M', u'𐐼'), + (0x10415, 'M', u'𐐽'), + (0x10416, 'M', u'𐐾'), + (0x10417, 'M', u'𐐿'), + (0x10418, 'M', u'𐑀'), + (0x10419, 'M', u'𐑁'), + (0x1041A, 'M', u'𐑂'), + (0x1041B, 'M', u'𐑃'), + (0x1041C, 'M', u'𐑄'), + (0x1041D, 'M', u'𐑅'), + (0x1041E, 'M', u'𐑆'), + (0x1041F, 'M', u'𐑇'), + (0x10420, 'M', u'𐑈'), + (0x10421, 'M', u'𐑉'), + (0x10422, 'M', u'𐑊'), + (0x10423, 'M', u'𐑋'), + (0x10424, 'M', u'𐑌'), + (0x10425, 'M', u'𐑍'), + (0x10426, 'M', u'𐑎'), + (0x10427, 'M', u'𐑏'), + (0x10428, 'V'), + (0x1049E, 'X'), + (0x104A0, 'V'), + (0x104AA, 'X'), + (0x104B0, 'M', u'𐓘'), + (0x104B1, 'M', u'𐓙'), + (0x104B2, 'M', u'𐓚'), + (0x104B3, 'M', u'𐓛'), + (0x104B4, 'M', u'𐓜'), + (0x104B5, 'M', u'𐓝'), + (0x104B6, 'M', u'𐓞'), + (0x104B7, 'M', u'𐓟'), + (0x104B8, 'M', u'𐓠'), + (0x104B9, 'M', u'𐓡'), + (0x104BA, 'M', u'𐓢'), + (0x104BB, 'M', u'𐓣'), + (0x104BC, 'M', u'𐓤'), + (0x104BD, 'M', u'𐓥'), + (0x104BE, 'M', u'𐓦'), + ] + +def _seg_54(): + return [ + (0x104BF, 'M', u'𐓧'), + (0x104C0, 'M', u'𐓨'), + (0x104C1, 'M', u'𐓩'), + (0x104C2, 'M', u'𐓪'), + (0x104C3, 'M', u'𐓫'), + (0x104C4, 'M', u'𐓬'), + (0x104C5, 'M', u'𐓭'), + (0x104C6, 'M', u'𐓮'), + (0x104C7, 'M', u'𐓯'), + (0x104C8, 'M', u'𐓰'), + (0x104C9, 'M', u'𐓱'), + (0x104CA, 'M', u'𐓲'), + (0x104CB, 'M', u'𐓳'), + (0x104CC, 'M', u'𐓴'), + (0x104CD, 'M', u'𐓵'), + (0x104CE, 'M', u'𐓶'), + (0x104CF, 'M', u'𐓷'), + (0x104D0, 'M', u'𐓸'), + (0x104D1, 'M', u'𐓹'), + (0x104D2, 'M', u'𐓺'), + (0x104D3, 'M', u'𐓻'), + (0x104D4, 'X'), + (0x104D8, 'V'), + (0x104FC, 'X'), + (0x10500, 'V'), + (0x10528, 'X'), + (0x10530, 'V'), + (0x10564, 'X'), + (0x1056F, 'V'), + (0x10570, 'X'), + (0x10600, 'V'), + (0x10737, 'X'), + (0x10740, 'V'), + (0x10756, 'X'), + (0x10760, 'V'), + (0x10768, 'X'), + (0x10800, 'V'), + (0x10806, 'X'), + (0x10808, 'V'), + (0x10809, 'X'), + (0x1080A, 'V'), + (0x10836, 'X'), + (0x10837, 'V'), + (0x10839, 'X'), + (0x1083C, 'V'), + (0x1083D, 'X'), + (0x1083F, 'V'), + (0x10856, 'X'), + (0x10857, 'V'), + (0x1089F, 'X'), + (0x108A7, 'V'), + (0x108B0, 'X'), + (0x108E0, 'V'), + (0x108F3, 'X'), + (0x108F4, 'V'), + (0x108F6, 'X'), + (0x108FB, 'V'), + (0x1091C, 'X'), + (0x1091F, 'V'), + (0x1093A, 'X'), + (0x1093F, 'V'), + (0x10940, 'X'), + (0x10980, 'V'), + (0x109B8, 'X'), + (0x109BC, 'V'), + (0x109D0, 'X'), + (0x109D2, 'V'), + (0x10A04, 'X'), + (0x10A05, 'V'), + (0x10A07, 'X'), + (0x10A0C, 'V'), + (0x10A14, 'X'), + (0x10A15, 'V'), + (0x10A18, 'X'), + (0x10A19, 'V'), + (0x10A36, 'X'), + (0x10A38, 'V'), + (0x10A3B, 'X'), + (0x10A3F, 'V'), + (0x10A49, 'X'), + (0x10A50, 'V'), + (0x10A59, 'X'), + (0x10A60, 'V'), + (0x10AA0, 'X'), + (0x10AC0, 'V'), + (0x10AE7, 'X'), + (0x10AEB, 'V'), + (0x10AF7, 'X'), + (0x10B00, 'V'), + (0x10B36, 'X'), + (0x10B39, 'V'), + (0x10B56, 'X'), + (0x10B58, 'V'), + (0x10B73, 'X'), + (0x10B78, 'V'), + (0x10B92, 'X'), + (0x10B99, 'V'), + (0x10B9D, 'X'), + (0x10BA9, 'V'), + (0x10BB0, 'X'), + ] + +def _seg_55(): + return [ + (0x10C00, 'V'), + (0x10C49, 'X'), + (0x10C80, 'M', u'𐳀'), + (0x10C81, 'M', u'𐳁'), + (0x10C82, 'M', u'𐳂'), + (0x10C83, 'M', u'𐳃'), + (0x10C84, 'M', u'𐳄'), + (0x10C85, 'M', u'𐳅'), + (0x10C86, 'M', u'𐳆'), + (0x10C87, 'M', u'𐳇'), + (0x10C88, 'M', u'𐳈'), + (0x10C89, 'M', u'𐳉'), + (0x10C8A, 'M', u'𐳊'), + (0x10C8B, 'M', u'𐳋'), + (0x10C8C, 'M', u'𐳌'), + (0x10C8D, 'M', u'𐳍'), + (0x10C8E, 'M', u'𐳎'), + (0x10C8F, 'M', u'𐳏'), + (0x10C90, 'M', u'𐳐'), + (0x10C91, 'M', u'𐳑'), + (0x10C92, 'M', u'𐳒'), + (0x10C93, 'M', u'𐳓'), + (0x10C94, 'M', u'𐳔'), + (0x10C95, 'M', u'𐳕'), + (0x10C96, 'M', u'𐳖'), + (0x10C97, 'M', u'𐳗'), + (0x10C98, 'M', u'𐳘'), + (0x10C99, 'M', u'𐳙'), + (0x10C9A, 'M', u'𐳚'), + (0x10C9B, 'M', u'𐳛'), + (0x10C9C, 'M', u'𐳜'), + (0x10C9D, 'M', u'𐳝'), + (0x10C9E, 'M', u'𐳞'), + (0x10C9F, 'M', u'𐳟'), + (0x10CA0, 'M', u'𐳠'), + (0x10CA1, 'M', u'𐳡'), + (0x10CA2, 'M', u'𐳢'), + (0x10CA3, 'M', u'𐳣'), + (0x10CA4, 'M', u'𐳤'), + (0x10CA5, 'M', u'𐳥'), + (0x10CA6, 'M', u'𐳦'), + (0x10CA7, 'M', u'𐳧'), + (0x10CA8, 'M', u'𐳨'), + (0x10CA9, 'M', u'𐳩'), + (0x10CAA, 'M', u'𐳪'), + (0x10CAB, 'M', u'𐳫'), + (0x10CAC, 'M', u'𐳬'), + (0x10CAD, 'M', u'𐳭'), + (0x10CAE, 'M', u'𐳮'), + (0x10CAF, 'M', u'𐳯'), + (0x10CB0, 'M', u'𐳰'), + (0x10CB1, 'M', u'𐳱'), + (0x10CB2, 'M', u'𐳲'), + (0x10CB3, 'X'), + (0x10CC0, 'V'), + (0x10CF3, 'X'), + (0x10CFA, 'V'), + (0x10D28, 'X'), + (0x10D30, 'V'), + (0x10D3A, 'X'), + (0x10E60, 'V'), + (0x10E7F, 'X'), + (0x10E80, 'V'), + (0x10EAA, 'X'), + (0x10EAB, 'V'), + (0x10EAE, 'X'), + (0x10EB0, 'V'), + (0x10EB2, 'X'), + (0x10F00, 'V'), + (0x10F28, 'X'), + (0x10F30, 'V'), + (0x10F5A, 'X'), + (0x10FB0, 'V'), + (0x10FCC, 'X'), + (0x10FE0, 'V'), + (0x10FF7, 'X'), + (0x11000, 'V'), + (0x1104E, 'X'), + (0x11052, 'V'), + (0x11070, 'X'), + (0x1107F, 'V'), + (0x110BD, 'X'), + (0x110BE, 'V'), + (0x110C2, 'X'), + (0x110D0, 'V'), + (0x110E9, 'X'), + (0x110F0, 'V'), + (0x110FA, 'X'), + (0x11100, 'V'), + (0x11135, 'X'), + (0x11136, 'V'), + (0x11148, 'X'), + (0x11150, 'V'), + (0x11177, 'X'), + (0x11180, 'V'), + (0x111E0, 'X'), + (0x111E1, 'V'), + (0x111F5, 'X'), + (0x11200, 'V'), + (0x11212, 'X'), + ] + +def _seg_56(): + return [ + (0x11213, 'V'), + (0x1123F, 'X'), + (0x11280, 'V'), + (0x11287, 'X'), + (0x11288, 'V'), + (0x11289, 'X'), + (0x1128A, 'V'), + (0x1128E, 'X'), + (0x1128F, 'V'), + (0x1129E, 'X'), + (0x1129F, 'V'), + (0x112AA, 'X'), + (0x112B0, 'V'), + (0x112EB, 'X'), + (0x112F0, 'V'), + (0x112FA, 'X'), + (0x11300, 'V'), + (0x11304, 'X'), + (0x11305, 'V'), + (0x1130D, 'X'), + (0x1130F, 'V'), + (0x11311, 'X'), + (0x11313, 'V'), + (0x11329, 'X'), + (0x1132A, 'V'), + (0x11331, 'X'), + (0x11332, 'V'), + (0x11334, 'X'), + (0x11335, 'V'), + (0x1133A, 'X'), + (0x1133B, 'V'), + (0x11345, 'X'), + (0x11347, 'V'), + (0x11349, 'X'), + (0x1134B, 'V'), + (0x1134E, 'X'), + (0x11350, 'V'), + (0x11351, 'X'), + (0x11357, 'V'), + (0x11358, 'X'), + (0x1135D, 'V'), + (0x11364, 'X'), + (0x11366, 'V'), + (0x1136D, 'X'), + (0x11370, 'V'), + (0x11375, 'X'), + (0x11400, 'V'), + (0x1145C, 'X'), + (0x1145D, 'V'), + (0x11462, 'X'), + (0x11480, 'V'), + (0x114C8, 'X'), + (0x114D0, 'V'), + (0x114DA, 'X'), + (0x11580, 'V'), + (0x115B6, 'X'), + (0x115B8, 'V'), + (0x115DE, 'X'), + (0x11600, 'V'), + (0x11645, 'X'), + (0x11650, 'V'), + (0x1165A, 'X'), + (0x11660, 'V'), + (0x1166D, 'X'), + (0x11680, 'V'), + (0x116B9, 'X'), + (0x116C0, 'V'), + (0x116CA, 'X'), + (0x11700, 'V'), + (0x1171B, 'X'), + (0x1171D, 'V'), + (0x1172C, 'X'), + (0x11730, 'V'), + (0x11740, 'X'), + (0x11800, 'V'), + (0x1183C, 'X'), + (0x118A0, 'M', u'𑣀'), + (0x118A1, 'M', u'𑣁'), + (0x118A2, 'M', u'𑣂'), + (0x118A3, 'M', u'𑣃'), + (0x118A4, 'M', u'𑣄'), + (0x118A5, 'M', u'𑣅'), + (0x118A6, 'M', u'𑣆'), + (0x118A7, 'M', u'𑣇'), + (0x118A8, 'M', u'𑣈'), + (0x118A9, 'M', u'𑣉'), + (0x118AA, 'M', u'𑣊'), + (0x118AB, 'M', u'𑣋'), + (0x118AC, 'M', u'𑣌'), + (0x118AD, 'M', u'𑣍'), + (0x118AE, 'M', u'𑣎'), + (0x118AF, 'M', u'𑣏'), + (0x118B0, 'M', u'𑣐'), + (0x118B1, 'M', u'𑣑'), + (0x118B2, 'M', u'𑣒'), + (0x118B3, 'M', u'𑣓'), + (0x118B4, 'M', u'𑣔'), + (0x118B5, 'M', u'𑣕'), + (0x118B6, 'M', u'𑣖'), + (0x118B7, 'M', u'𑣗'), + ] + +def _seg_57(): + return [ + (0x118B8, 'M', u'𑣘'), + (0x118B9, 'M', u'𑣙'), + (0x118BA, 'M', u'𑣚'), + (0x118BB, 'M', u'𑣛'), + (0x118BC, 'M', u'𑣜'), + (0x118BD, 'M', u'𑣝'), + (0x118BE, 'M', u'𑣞'), + (0x118BF, 'M', u'𑣟'), + (0x118C0, 'V'), + (0x118F3, 'X'), + (0x118FF, 'V'), + (0x11907, 'X'), + (0x11909, 'V'), + (0x1190A, 'X'), + (0x1190C, 'V'), + (0x11914, 'X'), + (0x11915, 'V'), + (0x11917, 'X'), + (0x11918, 'V'), + (0x11936, 'X'), + (0x11937, 'V'), + (0x11939, 'X'), + (0x1193B, 'V'), + (0x11947, 'X'), + (0x11950, 'V'), + (0x1195A, 'X'), + (0x119A0, 'V'), + (0x119A8, 'X'), + (0x119AA, 'V'), + (0x119D8, 'X'), + (0x119DA, 'V'), + (0x119E5, 'X'), + (0x11A00, 'V'), + (0x11A48, 'X'), + (0x11A50, 'V'), + (0x11AA3, 'X'), + (0x11AC0, 'V'), + (0x11AF9, 'X'), + (0x11C00, 'V'), + (0x11C09, 'X'), + (0x11C0A, 'V'), + (0x11C37, 'X'), + (0x11C38, 'V'), + (0x11C46, 'X'), + (0x11C50, 'V'), + (0x11C6D, 'X'), + (0x11C70, 'V'), + (0x11C90, 'X'), + (0x11C92, 'V'), + (0x11CA8, 'X'), + (0x11CA9, 'V'), + (0x11CB7, 'X'), + (0x11D00, 'V'), + (0x11D07, 'X'), + (0x11D08, 'V'), + (0x11D0A, 'X'), + (0x11D0B, 'V'), + (0x11D37, 'X'), + (0x11D3A, 'V'), + (0x11D3B, 'X'), + (0x11D3C, 'V'), + (0x11D3E, 'X'), + (0x11D3F, 'V'), + (0x11D48, 'X'), + (0x11D50, 'V'), + (0x11D5A, 'X'), + (0x11D60, 'V'), + (0x11D66, 'X'), + (0x11D67, 'V'), + (0x11D69, 'X'), + (0x11D6A, 'V'), + (0x11D8F, 'X'), + (0x11D90, 'V'), + (0x11D92, 'X'), + (0x11D93, 'V'), + (0x11D99, 'X'), + (0x11DA0, 'V'), + (0x11DAA, 'X'), + (0x11EE0, 'V'), + (0x11EF9, 'X'), + (0x11FB0, 'V'), + (0x11FB1, 'X'), + (0x11FC0, 'V'), + (0x11FF2, 'X'), + (0x11FFF, 'V'), + (0x1239A, 'X'), + (0x12400, 'V'), + (0x1246F, 'X'), + (0x12470, 'V'), + (0x12475, 'X'), + (0x12480, 'V'), + (0x12544, 'X'), + (0x13000, 'V'), + (0x1342F, 'X'), + (0x14400, 'V'), + (0x14647, 'X'), + (0x16800, 'V'), + (0x16A39, 'X'), + (0x16A40, 'V'), + (0x16A5F, 'X'), + ] + +def _seg_58(): + return [ + (0x16A60, 'V'), + (0x16A6A, 'X'), + (0x16A6E, 'V'), + (0x16A70, 'X'), + (0x16AD0, 'V'), + (0x16AEE, 'X'), + (0x16AF0, 'V'), + (0x16AF6, 'X'), + (0x16B00, 'V'), + (0x16B46, 'X'), + (0x16B50, 'V'), + (0x16B5A, 'X'), + (0x16B5B, 'V'), + (0x16B62, 'X'), + (0x16B63, 'V'), + (0x16B78, 'X'), + (0x16B7D, 'V'), + (0x16B90, 'X'), + (0x16E40, 'M', u'𖹠'), + (0x16E41, 'M', u'𖹡'), + (0x16E42, 'M', u'𖹢'), + (0x16E43, 'M', u'𖹣'), + (0x16E44, 'M', u'𖹤'), + (0x16E45, 'M', u'𖹥'), + (0x16E46, 'M', u'𖹦'), + (0x16E47, 'M', u'𖹧'), + (0x16E48, 'M', u'𖹨'), + (0x16E49, 'M', u'𖹩'), + (0x16E4A, 'M', u'𖹪'), + (0x16E4B, 'M', u'𖹫'), + (0x16E4C, 'M', u'𖹬'), + (0x16E4D, 'M', u'𖹭'), + (0x16E4E, 'M', u'𖹮'), + (0x16E4F, 'M', u'𖹯'), + (0x16E50, 'M', u'𖹰'), + (0x16E51, 'M', u'𖹱'), + (0x16E52, 'M', u'𖹲'), + (0x16E53, 'M', u'𖹳'), + (0x16E54, 'M', u'𖹴'), + (0x16E55, 'M', u'𖹵'), + (0x16E56, 'M', u'𖹶'), + (0x16E57, 'M', u'𖹷'), + (0x16E58, 'M', u'𖹸'), + (0x16E59, 'M', u'𖹹'), + (0x16E5A, 'M', u'𖹺'), + (0x16E5B, 'M', u'𖹻'), + (0x16E5C, 'M', u'𖹼'), + (0x16E5D, 'M', u'𖹽'), + (0x16E5E, 'M', u'𖹾'), + (0x16E5F, 'M', u'𖹿'), + (0x16E60, 'V'), + (0x16E9B, 'X'), + (0x16F00, 'V'), + (0x16F4B, 'X'), + (0x16F4F, 'V'), + (0x16F88, 'X'), + (0x16F8F, 'V'), + (0x16FA0, 'X'), + (0x16FE0, 'V'), + (0x16FE5, 'X'), + (0x16FF0, 'V'), + (0x16FF2, 'X'), + (0x17000, 'V'), + (0x187F8, 'X'), + (0x18800, 'V'), + (0x18CD6, 'X'), + (0x18D00, 'V'), + (0x18D09, 'X'), + (0x1B000, 'V'), + (0x1B11F, 'X'), + (0x1B150, 'V'), + (0x1B153, 'X'), + (0x1B164, 'V'), + (0x1B168, 'X'), + (0x1B170, 'V'), + (0x1B2FC, 'X'), + (0x1BC00, 'V'), + (0x1BC6B, 'X'), + (0x1BC70, 'V'), + (0x1BC7D, 'X'), + (0x1BC80, 'V'), + (0x1BC89, 'X'), + (0x1BC90, 'V'), + (0x1BC9A, 'X'), + (0x1BC9C, 'V'), + (0x1BCA0, 'I'), + (0x1BCA4, 'X'), + (0x1D000, 'V'), + (0x1D0F6, 'X'), + (0x1D100, 'V'), + (0x1D127, 'X'), + (0x1D129, 'V'), + (0x1D15E, 'M', u'𝅗𝅥'), + (0x1D15F, 'M', u'𝅘𝅥'), + (0x1D160, 'M', u'𝅘𝅥𝅮'), + (0x1D161, 'M', u'𝅘𝅥𝅯'), + (0x1D162, 'M', u'𝅘𝅥𝅰'), + (0x1D163, 'M', u'𝅘𝅥𝅱'), + (0x1D164, 'M', u'𝅘𝅥𝅲'), + (0x1D165, 'V'), + ] + +def _seg_59(): + return [ + (0x1D173, 'X'), + (0x1D17B, 'V'), + (0x1D1BB, 'M', u'𝆹𝅥'), + (0x1D1BC, 'M', u'𝆺𝅥'), + (0x1D1BD, 'M', u'𝆹𝅥𝅮'), + (0x1D1BE, 'M', u'𝆺𝅥𝅮'), + (0x1D1BF, 'M', u'𝆹𝅥𝅯'), + (0x1D1C0, 'M', u'𝆺𝅥𝅯'), + (0x1D1C1, 'V'), + (0x1D1E9, 'X'), + (0x1D200, 'V'), + (0x1D246, 'X'), + (0x1D2E0, 'V'), + (0x1D2F4, 'X'), + (0x1D300, 'V'), + (0x1D357, 'X'), + (0x1D360, 'V'), + (0x1D379, 'X'), + (0x1D400, 'M', u'a'), + (0x1D401, 'M', u'b'), + (0x1D402, 'M', u'c'), + (0x1D403, 'M', u'd'), + (0x1D404, 'M', u'e'), + (0x1D405, 'M', u'f'), + (0x1D406, 'M', u'g'), + (0x1D407, 'M', u'h'), + (0x1D408, 'M', u'i'), + (0x1D409, 'M', u'j'), + (0x1D40A, 'M', u'k'), + (0x1D40B, 'M', u'l'), + (0x1D40C, 'M', u'm'), + (0x1D40D, 'M', u'n'), + (0x1D40E, 'M', u'o'), + (0x1D40F, 'M', u'p'), + (0x1D410, 'M', u'q'), + (0x1D411, 'M', u'r'), + (0x1D412, 'M', u's'), + (0x1D413, 'M', u't'), + (0x1D414, 'M', u'u'), + (0x1D415, 'M', u'v'), + (0x1D416, 'M', u'w'), + (0x1D417, 'M', u'x'), + (0x1D418, 'M', u'y'), + (0x1D419, 'M', u'z'), + (0x1D41A, 'M', u'a'), + (0x1D41B, 'M', u'b'), + (0x1D41C, 'M', u'c'), + (0x1D41D, 'M', u'd'), + (0x1D41E, 'M', u'e'), + (0x1D41F, 'M', u'f'), + (0x1D420, 'M', u'g'), + (0x1D421, 'M', u'h'), + (0x1D422, 'M', u'i'), + (0x1D423, 'M', u'j'), + (0x1D424, 'M', u'k'), + (0x1D425, 'M', u'l'), + (0x1D426, 'M', u'm'), + (0x1D427, 'M', u'n'), + (0x1D428, 'M', u'o'), + (0x1D429, 'M', u'p'), + (0x1D42A, 'M', u'q'), + (0x1D42B, 'M', u'r'), + (0x1D42C, 'M', u's'), + (0x1D42D, 'M', u't'), + (0x1D42E, 'M', u'u'), + (0x1D42F, 'M', u'v'), + (0x1D430, 'M', u'w'), + (0x1D431, 'M', u'x'), + (0x1D432, 'M', u'y'), + (0x1D433, 'M', u'z'), + (0x1D434, 'M', u'a'), + (0x1D435, 'M', u'b'), + (0x1D436, 'M', u'c'), + (0x1D437, 'M', u'd'), + (0x1D438, 'M', u'e'), + (0x1D439, 'M', u'f'), + (0x1D43A, 'M', u'g'), + (0x1D43B, 'M', u'h'), + (0x1D43C, 'M', u'i'), + (0x1D43D, 'M', u'j'), + (0x1D43E, 'M', u'k'), + (0x1D43F, 'M', u'l'), + (0x1D440, 'M', u'm'), + (0x1D441, 'M', u'n'), + (0x1D442, 'M', u'o'), + (0x1D443, 'M', u'p'), + (0x1D444, 'M', u'q'), + (0x1D445, 'M', u'r'), + (0x1D446, 'M', u's'), + (0x1D447, 'M', u't'), + (0x1D448, 'M', u'u'), + (0x1D449, 'M', u'v'), + (0x1D44A, 'M', u'w'), + (0x1D44B, 'M', u'x'), + (0x1D44C, 'M', u'y'), + (0x1D44D, 'M', u'z'), + (0x1D44E, 'M', u'a'), + (0x1D44F, 'M', u'b'), + (0x1D450, 'M', u'c'), + (0x1D451, 'M', u'd'), + ] + +def _seg_60(): + return [ + (0x1D452, 'M', u'e'), + (0x1D453, 'M', u'f'), + (0x1D454, 'M', u'g'), + (0x1D455, 'X'), + (0x1D456, 'M', u'i'), + (0x1D457, 'M', u'j'), + (0x1D458, 'M', u'k'), + (0x1D459, 'M', u'l'), + (0x1D45A, 'M', u'm'), + (0x1D45B, 'M', u'n'), + (0x1D45C, 'M', u'o'), + (0x1D45D, 'M', u'p'), + (0x1D45E, 'M', u'q'), + (0x1D45F, 'M', u'r'), + (0x1D460, 'M', u's'), + (0x1D461, 'M', u't'), + (0x1D462, 'M', u'u'), + (0x1D463, 'M', u'v'), + (0x1D464, 'M', u'w'), + (0x1D465, 'M', u'x'), + (0x1D466, 'M', u'y'), + (0x1D467, 'M', u'z'), + (0x1D468, 'M', u'a'), + (0x1D469, 'M', u'b'), + (0x1D46A, 'M', u'c'), + (0x1D46B, 'M', u'd'), + (0x1D46C, 'M', u'e'), + (0x1D46D, 'M', u'f'), + (0x1D46E, 'M', u'g'), + (0x1D46F, 'M', u'h'), + (0x1D470, 'M', u'i'), + (0x1D471, 'M', u'j'), + (0x1D472, 'M', u'k'), + (0x1D473, 'M', u'l'), + (0x1D474, 'M', u'm'), + (0x1D475, 'M', u'n'), + (0x1D476, 'M', u'o'), + (0x1D477, 'M', u'p'), + (0x1D478, 'M', u'q'), + (0x1D479, 'M', u'r'), + (0x1D47A, 'M', u's'), + (0x1D47B, 'M', u't'), + (0x1D47C, 'M', u'u'), + (0x1D47D, 'M', u'v'), + (0x1D47E, 'M', u'w'), + (0x1D47F, 'M', u'x'), + (0x1D480, 'M', u'y'), + (0x1D481, 'M', u'z'), + (0x1D482, 'M', u'a'), + (0x1D483, 'M', u'b'), + (0x1D484, 'M', u'c'), + (0x1D485, 'M', u'd'), + (0x1D486, 'M', u'e'), + (0x1D487, 'M', u'f'), + (0x1D488, 'M', u'g'), + (0x1D489, 'M', u'h'), + (0x1D48A, 'M', u'i'), + (0x1D48B, 'M', u'j'), + (0x1D48C, 'M', u'k'), + (0x1D48D, 'M', u'l'), + (0x1D48E, 'M', u'm'), + (0x1D48F, 'M', u'n'), + (0x1D490, 'M', u'o'), + (0x1D491, 'M', u'p'), + (0x1D492, 'M', u'q'), + (0x1D493, 'M', u'r'), + (0x1D494, 'M', u's'), + (0x1D495, 'M', u't'), + (0x1D496, 'M', u'u'), + (0x1D497, 'M', u'v'), + (0x1D498, 'M', u'w'), + (0x1D499, 'M', u'x'), + (0x1D49A, 'M', u'y'), + (0x1D49B, 'M', u'z'), + (0x1D49C, 'M', u'a'), + (0x1D49D, 'X'), + (0x1D49E, 'M', u'c'), + (0x1D49F, 'M', u'd'), + (0x1D4A0, 'X'), + (0x1D4A2, 'M', u'g'), + (0x1D4A3, 'X'), + (0x1D4A5, 'M', u'j'), + (0x1D4A6, 'M', u'k'), + (0x1D4A7, 'X'), + (0x1D4A9, 'M', u'n'), + (0x1D4AA, 'M', u'o'), + (0x1D4AB, 'M', u'p'), + (0x1D4AC, 'M', u'q'), + (0x1D4AD, 'X'), + (0x1D4AE, 'M', u's'), + (0x1D4AF, 'M', u't'), + (0x1D4B0, 'M', u'u'), + (0x1D4B1, 'M', u'v'), + (0x1D4B2, 'M', u'w'), + (0x1D4B3, 'M', u'x'), + (0x1D4B4, 'M', u'y'), + (0x1D4B5, 'M', u'z'), + (0x1D4B6, 'M', u'a'), + (0x1D4B7, 'M', u'b'), + (0x1D4B8, 'M', u'c'), + ] + +def _seg_61(): + return [ + (0x1D4B9, 'M', u'd'), + (0x1D4BA, 'X'), + (0x1D4BB, 'M', u'f'), + (0x1D4BC, 'X'), + (0x1D4BD, 'M', u'h'), + (0x1D4BE, 'M', u'i'), + (0x1D4BF, 'M', u'j'), + (0x1D4C0, 'M', u'k'), + (0x1D4C1, 'M', u'l'), + (0x1D4C2, 'M', u'm'), + (0x1D4C3, 'M', u'n'), + (0x1D4C4, 'X'), + (0x1D4C5, 'M', u'p'), + (0x1D4C6, 'M', u'q'), + (0x1D4C7, 'M', u'r'), + (0x1D4C8, 'M', u's'), + (0x1D4C9, 'M', u't'), + (0x1D4CA, 'M', u'u'), + (0x1D4CB, 'M', u'v'), + (0x1D4CC, 'M', u'w'), + (0x1D4CD, 'M', u'x'), + (0x1D4CE, 'M', u'y'), + (0x1D4CF, 'M', u'z'), + (0x1D4D0, 'M', u'a'), + (0x1D4D1, 'M', u'b'), + (0x1D4D2, 'M', u'c'), + (0x1D4D3, 'M', u'd'), + (0x1D4D4, 'M', u'e'), + (0x1D4D5, 'M', u'f'), + (0x1D4D6, 'M', u'g'), + (0x1D4D7, 'M', u'h'), + (0x1D4D8, 'M', u'i'), + (0x1D4D9, 'M', u'j'), + (0x1D4DA, 'M', u'k'), + (0x1D4DB, 'M', u'l'), + (0x1D4DC, 'M', u'm'), + (0x1D4DD, 'M', u'n'), + (0x1D4DE, 'M', u'o'), + (0x1D4DF, 'M', u'p'), + (0x1D4E0, 'M', u'q'), + (0x1D4E1, 'M', u'r'), + (0x1D4E2, 'M', u's'), + (0x1D4E3, 'M', u't'), + (0x1D4E4, 'M', u'u'), + (0x1D4E5, 'M', u'v'), + (0x1D4E6, 'M', u'w'), + (0x1D4E7, 'M', u'x'), + (0x1D4E8, 'M', u'y'), + (0x1D4E9, 'M', u'z'), + (0x1D4EA, 'M', u'a'), + (0x1D4EB, 'M', u'b'), + (0x1D4EC, 'M', u'c'), + (0x1D4ED, 'M', u'd'), + (0x1D4EE, 'M', u'e'), + (0x1D4EF, 'M', u'f'), + (0x1D4F0, 'M', u'g'), + (0x1D4F1, 'M', u'h'), + (0x1D4F2, 'M', u'i'), + (0x1D4F3, 'M', u'j'), + (0x1D4F4, 'M', u'k'), + (0x1D4F5, 'M', u'l'), + (0x1D4F6, 'M', u'm'), + (0x1D4F7, 'M', u'n'), + (0x1D4F8, 'M', u'o'), + (0x1D4F9, 'M', u'p'), + (0x1D4FA, 'M', u'q'), + (0x1D4FB, 'M', u'r'), + (0x1D4FC, 'M', u's'), + (0x1D4FD, 'M', u't'), + (0x1D4FE, 'M', u'u'), + (0x1D4FF, 'M', u'v'), + (0x1D500, 'M', u'w'), + (0x1D501, 'M', u'x'), + (0x1D502, 'M', u'y'), + (0x1D503, 'M', u'z'), + (0x1D504, 'M', u'a'), + (0x1D505, 'M', u'b'), + (0x1D506, 'X'), + (0x1D507, 'M', u'd'), + (0x1D508, 'M', u'e'), + (0x1D509, 'M', u'f'), + (0x1D50A, 'M', u'g'), + (0x1D50B, 'X'), + (0x1D50D, 'M', u'j'), + (0x1D50E, 'M', u'k'), + (0x1D50F, 'M', u'l'), + (0x1D510, 'M', u'm'), + (0x1D511, 'M', u'n'), + (0x1D512, 'M', u'o'), + (0x1D513, 'M', u'p'), + (0x1D514, 'M', u'q'), + (0x1D515, 'X'), + (0x1D516, 'M', u's'), + (0x1D517, 'M', u't'), + (0x1D518, 'M', u'u'), + (0x1D519, 'M', u'v'), + (0x1D51A, 'M', u'w'), + (0x1D51B, 'M', u'x'), + (0x1D51C, 'M', u'y'), + (0x1D51D, 'X'), + ] + +def _seg_62(): + return [ + (0x1D51E, 'M', u'a'), + (0x1D51F, 'M', u'b'), + (0x1D520, 'M', u'c'), + (0x1D521, 'M', u'd'), + (0x1D522, 'M', u'e'), + (0x1D523, 'M', u'f'), + (0x1D524, 'M', u'g'), + (0x1D525, 'M', u'h'), + (0x1D526, 'M', u'i'), + (0x1D527, 'M', u'j'), + (0x1D528, 'M', u'k'), + (0x1D529, 'M', u'l'), + (0x1D52A, 'M', u'm'), + (0x1D52B, 'M', u'n'), + (0x1D52C, 'M', u'o'), + (0x1D52D, 'M', u'p'), + (0x1D52E, 'M', u'q'), + (0x1D52F, 'M', u'r'), + (0x1D530, 'M', u's'), + (0x1D531, 'M', u't'), + (0x1D532, 'M', u'u'), + (0x1D533, 'M', u'v'), + (0x1D534, 'M', u'w'), + (0x1D535, 'M', u'x'), + (0x1D536, 'M', u'y'), + (0x1D537, 'M', u'z'), + (0x1D538, 'M', u'a'), + (0x1D539, 'M', u'b'), + (0x1D53A, 'X'), + (0x1D53B, 'M', u'd'), + (0x1D53C, 'M', u'e'), + (0x1D53D, 'M', u'f'), + (0x1D53E, 'M', u'g'), + (0x1D53F, 'X'), + (0x1D540, 'M', u'i'), + (0x1D541, 'M', u'j'), + (0x1D542, 'M', u'k'), + (0x1D543, 'M', u'l'), + (0x1D544, 'M', u'm'), + (0x1D545, 'X'), + (0x1D546, 'M', u'o'), + (0x1D547, 'X'), + (0x1D54A, 'M', u's'), + (0x1D54B, 'M', u't'), + (0x1D54C, 'M', u'u'), + (0x1D54D, 'M', u'v'), + (0x1D54E, 'M', u'w'), + (0x1D54F, 'M', u'x'), + (0x1D550, 'M', u'y'), + (0x1D551, 'X'), + (0x1D552, 'M', u'a'), + (0x1D553, 'M', u'b'), + (0x1D554, 'M', u'c'), + (0x1D555, 'M', u'd'), + (0x1D556, 'M', u'e'), + (0x1D557, 'M', u'f'), + (0x1D558, 'M', u'g'), + (0x1D559, 'M', u'h'), + (0x1D55A, 'M', u'i'), + (0x1D55B, 'M', u'j'), + (0x1D55C, 'M', u'k'), + (0x1D55D, 'M', u'l'), + (0x1D55E, 'M', u'm'), + (0x1D55F, 'M', u'n'), + (0x1D560, 'M', u'o'), + (0x1D561, 'M', u'p'), + (0x1D562, 'M', u'q'), + (0x1D563, 'M', u'r'), + (0x1D564, 'M', u's'), + (0x1D565, 'M', u't'), + (0x1D566, 'M', u'u'), + (0x1D567, 'M', u'v'), + (0x1D568, 'M', u'w'), + (0x1D569, 'M', u'x'), + (0x1D56A, 'M', u'y'), + (0x1D56B, 'M', u'z'), + (0x1D56C, 'M', u'a'), + (0x1D56D, 'M', u'b'), + (0x1D56E, 'M', u'c'), + (0x1D56F, 'M', u'd'), + (0x1D570, 'M', u'e'), + (0x1D571, 'M', u'f'), + (0x1D572, 'M', u'g'), + (0x1D573, 'M', u'h'), + (0x1D574, 'M', u'i'), + (0x1D575, 'M', u'j'), + (0x1D576, 'M', u'k'), + (0x1D577, 'M', u'l'), + (0x1D578, 'M', u'm'), + (0x1D579, 'M', u'n'), + (0x1D57A, 'M', u'o'), + (0x1D57B, 'M', u'p'), + (0x1D57C, 'M', u'q'), + (0x1D57D, 'M', u'r'), + (0x1D57E, 'M', u's'), + (0x1D57F, 'M', u't'), + (0x1D580, 'M', u'u'), + (0x1D581, 'M', u'v'), + (0x1D582, 'M', u'w'), + (0x1D583, 'M', u'x'), + ] + +def _seg_63(): + return [ + (0x1D584, 'M', u'y'), + (0x1D585, 'M', u'z'), + (0x1D586, 'M', u'a'), + (0x1D587, 'M', u'b'), + (0x1D588, 'M', u'c'), + (0x1D589, 'M', u'd'), + (0x1D58A, 'M', u'e'), + (0x1D58B, 'M', u'f'), + (0x1D58C, 'M', u'g'), + (0x1D58D, 'M', u'h'), + (0x1D58E, 'M', u'i'), + (0x1D58F, 'M', u'j'), + (0x1D590, 'M', u'k'), + (0x1D591, 'M', u'l'), + (0x1D592, 'M', u'm'), + (0x1D593, 'M', u'n'), + (0x1D594, 'M', u'o'), + (0x1D595, 'M', u'p'), + (0x1D596, 'M', u'q'), + (0x1D597, 'M', u'r'), + (0x1D598, 'M', u's'), + (0x1D599, 'M', u't'), + (0x1D59A, 'M', u'u'), + (0x1D59B, 'M', u'v'), + (0x1D59C, 'M', u'w'), + (0x1D59D, 'M', u'x'), + (0x1D59E, 'M', u'y'), + (0x1D59F, 'M', u'z'), + (0x1D5A0, 'M', u'a'), + (0x1D5A1, 'M', u'b'), + (0x1D5A2, 'M', u'c'), + (0x1D5A3, 'M', u'd'), + (0x1D5A4, 'M', u'e'), + (0x1D5A5, 'M', u'f'), + (0x1D5A6, 'M', u'g'), + (0x1D5A7, 'M', u'h'), + (0x1D5A8, 'M', u'i'), + (0x1D5A9, 'M', u'j'), + (0x1D5AA, 'M', u'k'), + (0x1D5AB, 'M', u'l'), + (0x1D5AC, 'M', u'm'), + (0x1D5AD, 'M', u'n'), + (0x1D5AE, 'M', u'o'), + (0x1D5AF, 'M', u'p'), + (0x1D5B0, 'M', u'q'), + (0x1D5B1, 'M', u'r'), + (0x1D5B2, 'M', u's'), + (0x1D5B3, 'M', u't'), + (0x1D5B4, 'M', u'u'), + (0x1D5B5, 'M', u'v'), + (0x1D5B6, 'M', u'w'), + (0x1D5B7, 'M', u'x'), + (0x1D5B8, 'M', u'y'), + (0x1D5B9, 'M', u'z'), + (0x1D5BA, 'M', u'a'), + (0x1D5BB, 'M', u'b'), + (0x1D5BC, 'M', u'c'), + (0x1D5BD, 'M', u'd'), + (0x1D5BE, 'M', u'e'), + (0x1D5BF, 'M', u'f'), + (0x1D5C0, 'M', u'g'), + (0x1D5C1, 'M', u'h'), + (0x1D5C2, 'M', u'i'), + (0x1D5C3, 'M', u'j'), + (0x1D5C4, 'M', u'k'), + (0x1D5C5, 'M', u'l'), + (0x1D5C6, 'M', u'm'), + (0x1D5C7, 'M', u'n'), + (0x1D5C8, 'M', u'o'), + (0x1D5C9, 'M', u'p'), + (0x1D5CA, 'M', u'q'), + (0x1D5CB, 'M', u'r'), + (0x1D5CC, 'M', u's'), + (0x1D5CD, 'M', u't'), + (0x1D5CE, 'M', u'u'), + (0x1D5CF, 'M', u'v'), + (0x1D5D0, 'M', u'w'), + (0x1D5D1, 'M', u'x'), + (0x1D5D2, 'M', u'y'), + (0x1D5D3, 'M', u'z'), + (0x1D5D4, 'M', u'a'), + (0x1D5D5, 'M', u'b'), + (0x1D5D6, 'M', u'c'), + (0x1D5D7, 'M', u'd'), + (0x1D5D8, 'M', u'e'), + (0x1D5D9, 'M', u'f'), + (0x1D5DA, 'M', u'g'), + (0x1D5DB, 'M', u'h'), + (0x1D5DC, 'M', u'i'), + (0x1D5DD, 'M', u'j'), + (0x1D5DE, 'M', u'k'), + (0x1D5DF, 'M', u'l'), + (0x1D5E0, 'M', u'm'), + (0x1D5E1, 'M', u'n'), + (0x1D5E2, 'M', u'o'), + (0x1D5E3, 'M', u'p'), + (0x1D5E4, 'M', u'q'), + (0x1D5E5, 'M', u'r'), + (0x1D5E6, 'M', u's'), + (0x1D5E7, 'M', u't'), + ] + +def _seg_64(): + return [ + (0x1D5E8, 'M', u'u'), + (0x1D5E9, 'M', u'v'), + (0x1D5EA, 'M', u'w'), + (0x1D5EB, 'M', u'x'), + (0x1D5EC, 'M', u'y'), + (0x1D5ED, 'M', u'z'), + (0x1D5EE, 'M', u'a'), + (0x1D5EF, 'M', u'b'), + (0x1D5F0, 'M', u'c'), + (0x1D5F1, 'M', u'd'), + (0x1D5F2, 'M', u'e'), + (0x1D5F3, 'M', u'f'), + (0x1D5F4, 'M', u'g'), + (0x1D5F5, 'M', u'h'), + (0x1D5F6, 'M', u'i'), + (0x1D5F7, 'M', u'j'), + (0x1D5F8, 'M', u'k'), + (0x1D5F9, 'M', u'l'), + (0x1D5FA, 'M', u'm'), + (0x1D5FB, 'M', u'n'), + (0x1D5FC, 'M', u'o'), + (0x1D5FD, 'M', u'p'), + (0x1D5FE, 'M', u'q'), + (0x1D5FF, 'M', u'r'), + (0x1D600, 'M', u's'), + (0x1D601, 'M', u't'), + (0x1D602, 'M', u'u'), + (0x1D603, 'M', u'v'), + (0x1D604, 'M', u'w'), + (0x1D605, 'M', u'x'), + (0x1D606, 'M', u'y'), + (0x1D607, 'M', u'z'), + (0x1D608, 'M', u'a'), + (0x1D609, 'M', u'b'), + (0x1D60A, 'M', u'c'), + (0x1D60B, 'M', u'd'), + (0x1D60C, 'M', u'e'), + (0x1D60D, 'M', u'f'), + (0x1D60E, 'M', u'g'), + (0x1D60F, 'M', u'h'), + (0x1D610, 'M', u'i'), + (0x1D611, 'M', u'j'), + (0x1D612, 'M', u'k'), + (0x1D613, 'M', u'l'), + (0x1D614, 'M', u'm'), + (0x1D615, 'M', u'n'), + (0x1D616, 'M', u'o'), + (0x1D617, 'M', u'p'), + (0x1D618, 'M', u'q'), + (0x1D619, 'M', u'r'), + (0x1D61A, 'M', u's'), + (0x1D61B, 'M', u't'), + (0x1D61C, 'M', u'u'), + (0x1D61D, 'M', u'v'), + (0x1D61E, 'M', u'w'), + (0x1D61F, 'M', u'x'), + (0x1D620, 'M', u'y'), + (0x1D621, 'M', u'z'), + (0x1D622, 'M', u'a'), + (0x1D623, 'M', u'b'), + (0x1D624, 'M', u'c'), + (0x1D625, 'M', u'd'), + (0x1D626, 'M', u'e'), + (0x1D627, 'M', u'f'), + (0x1D628, 'M', u'g'), + (0x1D629, 'M', u'h'), + (0x1D62A, 'M', u'i'), + (0x1D62B, 'M', u'j'), + (0x1D62C, 'M', u'k'), + (0x1D62D, 'M', u'l'), + (0x1D62E, 'M', u'm'), + (0x1D62F, 'M', u'n'), + (0x1D630, 'M', u'o'), + (0x1D631, 'M', u'p'), + (0x1D632, 'M', u'q'), + (0x1D633, 'M', u'r'), + (0x1D634, 'M', u's'), + (0x1D635, 'M', u't'), + (0x1D636, 'M', u'u'), + (0x1D637, 'M', u'v'), + (0x1D638, 'M', u'w'), + (0x1D639, 'M', u'x'), + (0x1D63A, 'M', u'y'), + (0x1D63B, 'M', u'z'), + (0x1D63C, 'M', u'a'), + (0x1D63D, 'M', u'b'), + (0x1D63E, 'M', u'c'), + (0x1D63F, 'M', u'd'), + (0x1D640, 'M', u'e'), + (0x1D641, 'M', u'f'), + (0x1D642, 'M', u'g'), + (0x1D643, 'M', u'h'), + (0x1D644, 'M', u'i'), + (0x1D645, 'M', u'j'), + (0x1D646, 'M', u'k'), + (0x1D647, 'M', u'l'), + (0x1D648, 'M', u'm'), + (0x1D649, 'M', u'n'), + (0x1D64A, 'M', u'o'), + (0x1D64B, 'M', u'p'), + ] + +def _seg_65(): + return [ + (0x1D64C, 'M', u'q'), + (0x1D64D, 'M', u'r'), + (0x1D64E, 'M', u's'), + (0x1D64F, 'M', u't'), + (0x1D650, 'M', u'u'), + (0x1D651, 'M', u'v'), + (0x1D652, 'M', u'w'), + (0x1D653, 'M', u'x'), + (0x1D654, 'M', u'y'), + (0x1D655, 'M', u'z'), + (0x1D656, 'M', u'a'), + (0x1D657, 'M', u'b'), + (0x1D658, 'M', u'c'), + (0x1D659, 'M', u'd'), + (0x1D65A, 'M', u'e'), + (0x1D65B, 'M', u'f'), + (0x1D65C, 'M', u'g'), + (0x1D65D, 'M', u'h'), + (0x1D65E, 'M', u'i'), + (0x1D65F, 'M', u'j'), + (0x1D660, 'M', u'k'), + (0x1D661, 'M', u'l'), + (0x1D662, 'M', u'm'), + (0x1D663, 'M', u'n'), + (0x1D664, 'M', u'o'), + (0x1D665, 'M', u'p'), + (0x1D666, 'M', u'q'), + (0x1D667, 'M', u'r'), + (0x1D668, 'M', u's'), + (0x1D669, 'M', u't'), + (0x1D66A, 'M', u'u'), + (0x1D66B, 'M', u'v'), + (0x1D66C, 'M', u'w'), + (0x1D66D, 'M', u'x'), + (0x1D66E, 'M', u'y'), + (0x1D66F, 'M', u'z'), + (0x1D670, 'M', u'a'), + (0x1D671, 'M', u'b'), + (0x1D672, 'M', u'c'), + (0x1D673, 'M', u'd'), + (0x1D674, 'M', u'e'), + (0x1D675, 'M', u'f'), + (0x1D676, 'M', u'g'), + (0x1D677, 'M', u'h'), + (0x1D678, 'M', u'i'), + (0x1D679, 'M', u'j'), + (0x1D67A, 'M', u'k'), + (0x1D67B, 'M', u'l'), + (0x1D67C, 'M', u'm'), + (0x1D67D, 'M', u'n'), + (0x1D67E, 'M', u'o'), + (0x1D67F, 'M', u'p'), + (0x1D680, 'M', u'q'), + (0x1D681, 'M', u'r'), + (0x1D682, 'M', u's'), + (0x1D683, 'M', u't'), + (0x1D684, 'M', u'u'), + (0x1D685, 'M', u'v'), + (0x1D686, 'M', u'w'), + (0x1D687, 'M', u'x'), + (0x1D688, 'M', u'y'), + (0x1D689, 'M', u'z'), + (0x1D68A, 'M', u'a'), + (0x1D68B, 'M', u'b'), + (0x1D68C, 'M', u'c'), + (0x1D68D, 'M', u'd'), + (0x1D68E, 'M', u'e'), + (0x1D68F, 'M', u'f'), + (0x1D690, 'M', u'g'), + (0x1D691, 'M', u'h'), + (0x1D692, 'M', u'i'), + (0x1D693, 'M', u'j'), + (0x1D694, 'M', u'k'), + (0x1D695, 'M', u'l'), + (0x1D696, 'M', u'm'), + (0x1D697, 'M', u'n'), + (0x1D698, 'M', u'o'), + (0x1D699, 'M', u'p'), + (0x1D69A, 'M', u'q'), + (0x1D69B, 'M', u'r'), + (0x1D69C, 'M', u's'), + (0x1D69D, 'M', u't'), + (0x1D69E, 'M', u'u'), + (0x1D69F, 'M', u'v'), + (0x1D6A0, 'M', u'w'), + (0x1D6A1, 'M', u'x'), + (0x1D6A2, 'M', u'y'), + (0x1D6A3, 'M', u'z'), + (0x1D6A4, 'M', u'ı'), + (0x1D6A5, 'M', u'ȷ'), + (0x1D6A6, 'X'), + (0x1D6A8, 'M', u'α'), + (0x1D6A9, 'M', u'β'), + (0x1D6AA, 'M', u'γ'), + (0x1D6AB, 'M', u'δ'), + (0x1D6AC, 'M', u'ε'), + (0x1D6AD, 'M', u'ζ'), + (0x1D6AE, 'M', u'η'), + (0x1D6AF, 'M', u'θ'), + (0x1D6B0, 'M', u'ι'), + ] + +def _seg_66(): + return [ + (0x1D6B1, 'M', u'κ'), + (0x1D6B2, 'M', u'λ'), + (0x1D6B3, 'M', u'μ'), + (0x1D6B4, 'M', u'ν'), + (0x1D6B5, 'M', u'ξ'), + (0x1D6B6, 'M', u'ο'), + (0x1D6B7, 'M', u'π'), + (0x1D6B8, 'M', u'ρ'), + (0x1D6B9, 'M', u'θ'), + (0x1D6BA, 'M', u'σ'), + (0x1D6BB, 'M', u'τ'), + (0x1D6BC, 'M', u'υ'), + (0x1D6BD, 'M', u'φ'), + (0x1D6BE, 'M', u'χ'), + (0x1D6BF, 'M', u'ψ'), + (0x1D6C0, 'M', u'ω'), + (0x1D6C1, 'M', u'∇'), + (0x1D6C2, 'M', u'α'), + (0x1D6C3, 'M', u'β'), + (0x1D6C4, 'M', u'γ'), + (0x1D6C5, 'M', u'δ'), + (0x1D6C6, 'M', u'ε'), + (0x1D6C7, 'M', u'ζ'), + (0x1D6C8, 'M', u'η'), + (0x1D6C9, 'M', u'θ'), + (0x1D6CA, 'M', u'ι'), + (0x1D6CB, 'M', u'κ'), + (0x1D6CC, 'M', u'λ'), + (0x1D6CD, 'M', u'μ'), + (0x1D6CE, 'M', u'ν'), + (0x1D6CF, 'M', u'ξ'), + (0x1D6D0, 'M', u'ο'), + (0x1D6D1, 'M', u'π'), + (0x1D6D2, 'M', u'ρ'), + (0x1D6D3, 'M', u'σ'), + (0x1D6D5, 'M', u'τ'), + (0x1D6D6, 'M', u'υ'), + (0x1D6D7, 'M', u'φ'), + (0x1D6D8, 'M', u'χ'), + (0x1D6D9, 'M', u'ψ'), + (0x1D6DA, 'M', u'ω'), + (0x1D6DB, 'M', u'∂'), + (0x1D6DC, 'M', u'ε'), + (0x1D6DD, 'M', u'θ'), + (0x1D6DE, 'M', u'κ'), + (0x1D6DF, 'M', u'φ'), + (0x1D6E0, 'M', u'ρ'), + (0x1D6E1, 'M', u'π'), + (0x1D6E2, 'M', u'α'), + (0x1D6E3, 'M', u'β'), + (0x1D6E4, 'M', u'γ'), + (0x1D6E5, 'M', u'δ'), + (0x1D6E6, 'M', u'ε'), + (0x1D6E7, 'M', u'ζ'), + (0x1D6E8, 'M', u'η'), + (0x1D6E9, 'M', u'θ'), + (0x1D6EA, 'M', u'ι'), + (0x1D6EB, 'M', u'κ'), + (0x1D6EC, 'M', u'λ'), + (0x1D6ED, 'M', u'μ'), + (0x1D6EE, 'M', u'ν'), + (0x1D6EF, 'M', u'ξ'), + (0x1D6F0, 'M', u'ο'), + (0x1D6F1, 'M', u'π'), + (0x1D6F2, 'M', u'ρ'), + (0x1D6F3, 'M', u'θ'), + (0x1D6F4, 'M', u'σ'), + (0x1D6F5, 'M', u'τ'), + (0x1D6F6, 'M', u'υ'), + (0x1D6F7, 'M', u'φ'), + (0x1D6F8, 'M', u'χ'), + (0x1D6F9, 'M', u'ψ'), + (0x1D6FA, 'M', u'ω'), + (0x1D6FB, 'M', u'∇'), + (0x1D6FC, 'M', u'α'), + (0x1D6FD, 'M', u'β'), + (0x1D6FE, 'M', u'γ'), + (0x1D6FF, 'M', u'δ'), + (0x1D700, 'M', u'ε'), + (0x1D701, 'M', u'ζ'), + (0x1D702, 'M', u'η'), + (0x1D703, 'M', u'θ'), + (0x1D704, 'M', u'ι'), + (0x1D705, 'M', u'κ'), + (0x1D706, 'M', u'λ'), + (0x1D707, 'M', u'μ'), + (0x1D708, 'M', u'ν'), + (0x1D709, 'M', u'ξ'), + (0x1D70A, 'M', u'ο'), + (0x1D70B, 'M', u'π'), + (0x1D70C, 'M', u'ρ'), + (0x1D70D, 'M', u'σ'), + (0x1D70F, 'M', u'τ'), + (0x1D710, 'M', u'υ'), + (0x1D711, 'M', u'φ'), + (0x1D712, 'M', u'χ'), + (0x1D713, 'M', u'ψ'), + (0x1D714, 'M', u'ω'), + (0x1D715, 'M', u'∂'), + (0x1D716, 'M', u'ε'), + ] + +def _seg_67(): + return [ + (0x1D717, 'M', u'θ'), + (0x1D718, 'M', u'κ'), + (0x1D719, 'M', u'φ'), + (0x1D71A, 'M', u'ρ'), + (0x1D71B, 'M', u'π'), + (0x1D71C, 'M', u'α'), + (0x1D71D, 'M', u'β'), + (0x1D71E, 'M', u'γ'), + (0x1D71F, 'M', u'δ'), + (0x1D720, 'M', u'ε'), + (0x1D721, 'M', u'ζ'), + (0x1D722, 'M', u'η'), + (0x1D723, 'M', u'θ'), + (0x1D724, 'M', u'ι'), + (0x1D725, 'M', u'κ'), + (0x1D726, 'M', u'λ'), + (0x1D727, 'M', u'μ'), + (0x1D728, 'M', u'ν'), + (0x1D729, 'M', u'ξ'), + (0x1D72A, 'M', u'ο'), + (0x1D72B, 'M', u'π'), + (0x1D72C, 'M', u'ρ'), + (0x1D72D, 'M', u'θ'), + (0x1D72E, 'M', u'σ'), + (0x1D72F, 'M', u'τ'), + (0x1D730, 'M', u'υ'), + (0x1D731, 'M', u'φ'), + (0x1D732, 'M', u'χ'), + (0x1D733, 'M', u'ψ'), + (0x1D734, 'M', u'ω'), + (0x1D735, 'M', u'∇'), + (0x1D736, 'M', u'α'), + (0x1D737, 'M', u'β'), + (0x1D738, 'M', u'γ'), + (0x1D739, 'M', u'δ'), + (0x1D73A, 'M', u'ε'), + (0x1D73B, 'M', u'ζ'), + (0x1D73C, 'M', u'η'), + (0x1D73D, 'M', u'θ'), + (0x1D73E, 'M', u'ι'), + (0x1D73F, 'M', u'κ'), + (0x1D740, 'M', u'λ'), + (0x1D741, 'M', u'μ'), + (0x1D742, 'M', u'ν'), + (0x1D743, 'M', u'ξ'), + (0x1D744, 'M', u'ο'), + (0x1D745, 'M', u'π'), + (0x1D746, 'M', u'ρ'), + (0x1D747, 'M', u'σ'), + (0x1D749, 'M', u'τ'), + (0x1D74A, 'M', u'υ'), + (0x1D74B, 'M', u'φ'), + (0x1D74C, 'M', u'χ'), + (0x1D74D, 'M', u'ψ'), + (0x1D74E, 'M', u'ω'), + (0x1D74F, 'M', u'∂'), + (0x1D750, 'M', u'ε'), + (0x1D751, 'M', u'θ'), + (0x1D752, 'M', u'κ'), + (0x1D753, 'M', u'φ'), + (0x1D754, 'M', u'ρ'), + (0x1D755, 'M', u'π'), + (0x1D756, 'M', u'α'), + (0x1D757, 'M', u'β'), + (0x1D758, 'M', u'γ'), + (0x1D759, 'M', u'δ'), + (0x1D75A, 'M', u'ε'), + (0x1D75B, 'M', u'ζ'), + (0x1D75C, 'M', u'η'), + (0x1D75D, 'M', u'θ'), + (0x1D75E, 'M', u'ι'), + (0x1D75F, 'M', u'κ'), + (0x1D760, 'M', u'λ'), + (0x1D761, 'M', u'μ'), + (0x1D762, 'M', u'ν'), + (0x1D763, 'M', u'ξ'), + (0x1D764, 'M', u'ο'), + (0x1D765, 'M', u'π'), + (0x1D766, 'M', u'ρ'), + (0x1D767, 'M', u'θ'), + (0x1D768, 'M', u'σ'), + (0x1D769, 'M', u'τ'), + (0x1D76A, 'M', u'υ'), + (0x1D76B, 'M', u'φ'), + (0x1D76C, 'M', u'χ'), + (0x1D76D, 'M', u'ψ'), + (0x1D76E, 'M', u'ω'), + (0x1D76F, 'M', u'∇'), + (0x1D770, 'M', u'α'), + (0x1D771, 'M', u'β'), + (0x1D772, 'M', u'γ'), + (0x1D773, 'M', u'δ'), + (0x1D774, 'M', u'ε'), + (0x1D775, 'M', u'ζ'), + (0x1D776, 'M', u'η'), + (0x1D777, 'M', u'θ'), + (0x1D778, 'M', u'ι'), + (0x1D779, 'M', u'κ'), + (0x1D77A, 'M', u'λ'), + (0x1D77B, 'M', u'μ'), + ] + +def _seg_68(): + return [ + (0x1D77C, 'M', u'ν'), + (0x1D77D, 'M', u'ξ'), + (0x1D77E, 'M', u'ο'), + (0x1D77F, 'M', u'π'), + (0x1D780, 'M', u'ρ'), + (0x1D781, 'M', u'σ'), + (0x1D783, 'M', u'τ'), + (0x1D784, 'M', u'υ'), + (0x1D785, 'M', u'φ'), + (0x1D786, 'M', u'χ'), + (0x1D787, 'M', u'ψ'), + (0x1D788, 'M', u'ω'), + (0x1D789, 'M', u'∂'), + (0x1D78A, 'M', u'ε'), + (0x1D78B, 'M', u'θ'), + (0x1D78C, 'M', u'κ'), + (0x1D78D, 'M', u'φ'), + (0x1D78E, 'M', u'ρ'), + (0x1D78F, 'M', u'π'), + (0x1D790, 'M', u'α'), + (0x1D791, 'M', u'β'), + (0x1D792, 'M', u'γ'), + (0x1D793, 'M', u'δ'), + (0x1D794, 'M', u'ε'), + (0x1D795, 'M', u'ζ'), + (0x1D796, 'M', u'η'), + (0x1D797, 'M', u'θ'), + (0x1D798, 'M', u'ι'), + (0x1D799, 'M', u'κ'), + (0x1D79A, 'M', u'λ'), + (0x1D79B, 'M', u'μ'), + (0x1D79C, 'M', u'ν'), + (0x1D79D, 'M', u'ξ'), + (0x1D79E, 'M', u'ο'), + (0x1D79F, 'M', u'π'), + (0x1D7A0, 'M', u'ρ'), + (0x1D7A1, 'M', u'θ'), + (0x1D7A2, 'M', u'σ'), + (0x1D7A3, 'M', u'τ'), + (0x1D7A4, 'M', u'υ'), + (0x1D7A5, 'M', u'φ'), + (0x1D7A6, 'M', u'χ'), + (0x1D7A7, 'M', u'ψ'), + (0x1D7A8, 'M', u'ω'), + (0x1D7A9, 'M', u'∇'), + (0x1D7AA, 'M', u'α'), + (0x1D7AB, 'M', u'β'), + (0x1D7AC, 'M', u'γ'), + (0x1D7AD, 'M', u'δ'), + (0x1D7AE, 'M', u'ε'), + (0x1D7AF, 'M', u'ζ'), + (0x1D7B0, 'M', u'η'), + (0x1D7B1, 'M', u'θ'), + (0x1D7B2, 'M', u'ι'), + (0x1D7B3, 'M', u'κ'), + (0x1D7B4, 'M', u'λ'), + (0x1D7B5, 'M', u'μ'), + (0x1D7B6, 'M', u'ν'), + (0x1D7B7, 'M', u'ξ'), + (0x1D7B8, 'M', u'ο'), + (0x1D7B9, 'M', u'π'), + (0x1D7BA, 'M', u'ρ'), + (0x1D7BB, 'M', u'σ'), + (0x1D7BD, 'M', u'τ'), + (0x1D7BE, 'M', u'υ'), + (0x1D7BF, 'M', u'φ'), + (0x1D7C0, 'M', u'χ'), + (0x1D7C1, 'M', u'ψ'), + (0x1D7C2, 'M', u'ω'), + (0x1D7C3, 'M', u'∂'), + (0x1D7C4, 'M', u'ε'), + (0x1D7C5, 'M', u'θ'), + (0x1D7C6, 'M', u'κ'), + (0x1D7C7, 'M', u'φ'), + (0x1D7C8, 'M', u'ρ'), + (0x1D7C9, 'M', u'π'), + (0x1D7CA, 'M', u'ϝ'), + (0x1D7CC, 'X'), + (0x1D7CE, 'M', u'0'), + (0x1D7CF, 'M', u'1'), + (0x1D7D0, 'M', u'2'), + (0x1D7D1, 'M', u'3'), + (0x1D7D2, 'M', u'4'), + (0x1D7D3, 'M', u'5'), + (0x1D7D4, 'M', u'6'), + (0x1D7D5, 'M', u'7'), + (0x1D7D6, 'M', u'8'), + (0x1D7D7, 'M', u'9'), + (0x1D7D8, 'M', u'0'), + (0x1D7D9, 'M', u'1'), + (0x1D7DA, 'M', u'2'), + (0x1D7DB, 'M', u'3'), + (0x1D7DC, 'M', u'4'), + (0x1D7DD, 'M', u'5'), + (0x1D7DE, 'M', u'6'), + (0x1D7DF, 'M', u'7'), + (0x1D7E0, 'M', u'8'), + (0x1D7E1, 'M', u'9'), + (0x1D7E2, 'M', u'0'), + (0x1D7E3, 'M', u'1'), + ] + +def _seg_69(): + return [ + (0x1D7E4, 'M', u'2'), + (0x1D7E5, 'M', u'3'), + (0x1D7E6, 'M', u'4'), + (0x1D7E7, 'M', u'5'), + (0x1D7E8, 'M', u'6'), + (0x1D7E9, 'M', u'7'), + (0x1D7EA, 'M', u'8'), + (0x1D7EB, 'M', u'9'), + (0x1D7EC, 'M', u'0'), + (0x1D7ED, 'M', u'1'), + (0x1D7EE, 'M', u'2'), + (0x1D7EF, 'M', u'3'), + (0x1D7F0, 'M', u'4'), + (0x1D7F1, 'M', u'5'), + (0x1D7F2, 'M', u'6'), + (0x1D7F3, 'M', u'7'), + (0x1D7F4, 'M', u'8'), + (0x1D7F5, 'M', u'9'), + (0x1D7F6, 'M', u'0'), + (0x1D7F7, 'M', u'1'), + (0x1D7F8, 'M', u'2'), + (0x1D7F9, 'M', u'3'), + (0x1D7FA, 'M', u'4'), + (0x1D7FB, 'M', u'5'), + (0x1D7FC, 'M', u'6'), + (0x1D7FD, 'M', u'7'), + (0x1D7FE, 'M', u'8'), + (0x1D7FF, 'M', u'9'), + (0x1D800, 'V'), + (0x1DA8C, 'X'), + (0x1DA9B, 'V'), + (0x1DAA0, 'X'), + (0x1DAA1, 'V'), + (0x1DAB0, 'X'), + (0x1E000, 'V'), + (0x1E007, 'X'), + (0x1E008, 'V'), + (0x1E019, 'X'), + (0x1E01B, 'V'), + (0x1E022, 'X'), + (0x1E023, 'V'), + (0x1E025, 'X'), + (0x1E026, 'V'), + (0x1E02B, 'X'), + (0x1E100, 'V'), + (0x1E12D, 'X'), + (0x1E130, 'V'), + (0x1E13E, 'X'), + (0x1E140, 'V'), + (0x1E14A, 'X'), + (0x1E14E, 'V'), + (0x1E150, 'X'), + (0x1E2C0, 'V'), + (0x1E2FA, 'X'), + (0x1E2FF, 'V'), + (0x1E300, 'X'), + (0x1E800, 'V'), + (0x1E8C5, 'X'), + (0x1E8C7, 'V'), + (0x1E8D7, 'X'), + (0x1E900, 'M', u'𞤢'), + (0x1E901, 'M', u'𞤣'), + (0x1E902, 'M', u'𞤤'), + (0x1E903, 'M', u'𞤥'), + (0x1E904, 'M', u'𞤦'), + (0x1E905, 'M', u'𞤧'), + (0x1E906, 'M', u'𞤨'), + (0x1E907, 'M', u'𞤩'), + (0x1E908, 'M', u'𞤪'), + (0x1E909, 'M', u'𞤫'), + (0x1E90A, 'M', u'𞤬'), + (0x1E90B, 'M', u'𞤭'), + (0x1E90C, 'M', u'𞤮'), + (0x1E90D, 'M', u'𞤯'), + (0x1E90E, 'M', u'𞤰'), + (0x1E90F, 'M', u'𞤱'), + (0x1E910, 'M', u'𞤲'), + (0x1E911, 'M', u'𞤳'), + (0x1E912, 'M', u'𞤴'), + (0x1E913, 'M', u'𞤵'), + (0x1E914, 'M', u'𞤶'), + (0x1E915, 'M', u'𞤷'), + (0x1E916, 'M', u'𞤸'), + (0x1E917, 'M', u'𞤹'), + (0x1E918, 'M', u'𞤺'), + (0x1E919, 'M', u'𞤻'), + (0x1E91A, 'M', u'𞤼'), + (0x1E91B, 'M', u'𞤽'), + (0x1E91C, 'M', u'𞤾'), + (0x1E91D, 'M', u'𞤿'), + (0x1E91E, 'M', u'𞥀'), + (0x1E91F, 'M', u'𞥁'), + (0x1E920, 'M', u'𞥂'), + (0x1E921, 'M', u'𞥃'), + (0x1E922, 'V'), + (0x1E94C, 'X'), + (0x1E950, 'V'), + (0x1E95A, 'X'), + (0x1E95E, 'V'), + (0x1E960, 'X'), + ] + +def _seg_70(): + return [ + (0x1EC71, 'V'), + (0x1ECB5, 'X'), + (0x1ED01, 'V'), + (0x1ED3E, 'X'), + (0x1EE00, 'M', u'ا'), + (0x1EE01, 'M', u'ب'), + (0x1EE02, 'M', u'ج'), + (0x1EE03, 'M', u'د'), + (0x1EE04, 'X'), + (0x1EE05, 'M', u'و'), + (0x1EE06, 'M', u'ز'), + (0x1EE07, 'M', u'ح'), + (0x1EE08, 'M', u'ط'), + (0x1EE09, 'M', u'ي'), + (0x1EE0A, 'M', u'ك'), + (0x1EE0B, 'M', u'ل'), + (0x1EE0C, 'M', u'م'), + (0x1EE0D, 'M', u'ن'), + (0x1EE0E, 'M', u'س'), + (0x1EE0F, 'M', u'ع'), + (0x1EE10, 'M', u'ف'), + (0x1EE11, 'M', u'ص'), + (0x1EE12, 'M', u'ق'), + (0x1EE13, 'M', u'ر'), + (0x1EE14, 'M', u'ش'), + (0x1EE15, 'M', u'ت'), + (0x1EE16, 'M', u'ث'), + (0x1EE17, 'M', u'خ'), + (0x1EE18, 'M', u'ذ'), + (0x1EE19, 'M', u'ض'), + (0x1EE1A, 'M', u'ظ'), + (0x1EE1B, 'M', u'غ'), + (0x1EE1C, 'M', u'ٮ'), + (0x1EE1D, 'M', u'ں'), + (0x1EE1E, 'M', u'ڡ'), + (0x1EE1F, 'M', u'ٯ'), + (0x1EE20, 'X'), + (0x1EE21, 'M', u'ب'), + (0x1EE22, 'M', u'ج'), + (0x1EE23, 'X'), + (0x1EE24, 'M', u'ه'), + (0x1EE25, 'X'), + (0x1EE27, 'M', u'ح'), + (0x1EE28, 'X'), + (0x1EE29, 'M', u'ي'), + (0x1EE2A, 'M', u'ك'), + (0x1EE2B, 'M', u'ل'), + (0x1EE2C, 'M', u'م'), + (0x1EE2D, 'M', u'ن'), + (0x1EE2E, 'M', u'س'), + (0x1EE2F, 'M', u'ع'), + (0x1EE30, 'M', u'ف'), + (0x1EE31, 'M', u'ص'), + (0x1EE32, 'M', u'ق'), + (0x1EE33, 'X'), + (0x1EE34, 'M', u'ش'), + (0x1EE35, 'M', u'ت'), + (0x1EE36, 'M', u'ث'), + (0x1EE37, 'M', u'خ'), + (0x1EE38, 'X'), + (0x1EE39, 'M', u'ض'), + (0x1EE3A, 'X'), + (0x1EE3B, 'M', u'غ'), + (0x1EE3C, 'X'), + (0x1EE42, 'M', u'ج'), + (0x1EE43, 'X'), + (0x1EE47, 'M', u'ح'), + (0x1EE48, 'X'), + (0x1EE49, 'M', u'ي'), + (0x1EE4A, 'X'), + (0x1EE4B, 'M', u'ل'), + (0x1EE4C, 'X'), + (0x1EE4D, 'M', u'ن'), + (0x1EE4E, 'M', u'س'), + (0x1EE4F, 'M', u'ع'), + (0x1EE50, 'X'), + (0x1EE51, 'M', u'ص'), + (0x1EE52, 'M', u'ق'), + (0x1EE53, 'X'), + (0x1EE54, 'M', u'ش'), + (0x1EE55, 'X'), + (0x1EE57, 'M', u'خ'), + (0x1EE58, 'X'), + (0x1EE59, 'M', u'ض'), + (0x1EE5A, 'X'), + (0x1EE5B, 'M', u'غ'), + (0x1EE5C, 'X'), + (0x1EE5D, 'M', u'ں'), + (0x1EE5E, 'X'), + (0x1EE5F, 'M', u'ٯ'), + (0x1EE60, 'X'), + (0x1EE61, 'M', u'ب'), + (0x1EE62, 'M', u'ج'), + (0x1EE63, 'X'), + (0x1EE64, 'M', u'ه'), + (0x1EE65, 'X'), + (0x1EE67, 'M', u'ح'), + (0x1EE68, 'M', u'ط'), + (0x1EE69, 'M', u'ي'), + (0x1EE6A, 'M', u'ك'), + ] + +def _seg_71(): + return [ + (0x1EE6B, 'X'), + (0x1EE6C, 'M', u'م'), + (0x1EE6D, 'M', u'ن'), + (0x1EE6E, 'M', u'س'), + (0x1EE6F, 'M', u'ع'), + (0x1EE70, 'M', u'ف'), + (0x1EE71, 'M', u'ص'), + (0x1EE72, 'M', u'ق'), + (0x1EE73, 'X'), + (0x1EE74, 'M', u'ش'), + (0x1EE75, 'M', u'ت'), + (0x1EE76, 'M', u'ث'), + (0x1EE77, 'M', u'خ'), + (0x1EE78, 'X'), + (0x1EE79, 'M', u'ض'), + (0x1EE7A, 'M', u'ظ'), + (0x1EE7B, 'M', u'غ'), + (0x1EE7C, 'M', u'ٮ'), + (0x1EE7D, 'X'), + (0x1EE7E, 'M', u'ڡ'), + (0x1EE7F, 'X'), + (0x1EE80, 'M', u'ا'), + (0x1EE81, 'M', u'ب'), + (0x1EE82, 'M', u'ج'), + (0x1EE83, 'M', u'د'), + (0x1EE84, 'M', u'ه'), + (0x1EE85, 'M', u'و'), + (0x1EE86, 'M', u'ز'), + (0x1EE87, 'M', u'ح'), + (0x1EE88, 'M', u'ط'), + (0x1EE89, 'M', u'ي'), + (0x1EE8A, 'X'), + (0x1EE8B, 'M', u'ل'), + (0x1EE8C, 'M', u'م'), + (0x1EE8D, 'M', u'ن'), + (0x1EE8E, 'M', u'س'), + (0x1EE8F, 'M', u'ع'), + (0x1EE90, 'M', u'ف'), + (0x1EE91, 'M', u'ص'), + (0x1EE92, 'M', u'ق'), + (0x1EE93, 'M', u'ر'), + (0x1EE94, 'M', u'ش'), + (0x1EE95, 'M', u'ت'), + (0x1EE96, 'M', u'ث'), + (0x1EE97, 'M', u'خ'), + (0x1EE98, 'M', u'ذ'), + (0x1EE99, 'M', u'ض'), + (0x1EE9A, 'M', u'ظ'), + (0x1EE9B, 'M', u'غ'), + (0x1EE9C, 'X'), + (0x1EEA1, 'M', u'ب'), + (0x1EEA2, 'M', u'ج'), + (0x1EEA3, 'M', u'د'), + (0x1EEA4, 'X'), + (0x1EEA5, 'M', u'و'), + (0x1EEA6, 'M', u'ز'), + (0x1EEA7, 'M', u'ح'), + (0x1EEA8, 'M', u'ط'), + (0x1EEA9, 'M', u'ي'), + (0x1EEAA, 'X'), + (0x1EEAB, 'M', u'ل'), + (0x1EEAC, 'M', u'م'), + (0x1EEAD, 'M', u'ن'), + (0x1EEAE, 'M', u'س'), + (0x1EEAF, 'M', u'ع'), + (0x1EEB0, 'M', u'ف'), + (0x1EEB1, 'M', u'ص'), + (0x1EEB2, 'M', u'ق'), + (0x1EEB3, 'M', u'ر'), + (0x1EEB4, 'M', u'ش'), + (0x1EEB5, 'M', u'ت'), + (0x1EEB6, 'M', u'ث'), + (0x1EEB7, 'M', u'خ'), + (0x1EEB8, 'M', u'ذ'), + (0x1EEB9, 'M', u'ض'), + (0x1EEBA, 'M', u'ظ'), + (0x1EEBB, 'M', u'غ'), + (0x1EEBC, 'X'), + (0x1EEF0, 'V'), + (0x1EEF2, 'X'), + (0x1F000, 'V'), + (0x1F02C, 'X'), + (0x1F030, 'V'), + (0x1F094, 'X'), + (0x1F0A0, 'V'), + (0x1F0AF, 'X'), + (0x1F0B1, 'V'), + (0x1F0C0, 'X'), + (0x1F0C1, 'V'), + (0x1F0D0, 'X'), + (0x1F0D1, 'V'), + (0x1F0F6, 'X'), + (0x1F101, '3', u'0,'), + (0x1F102, '3', u'1,'), + (0x1F103, '3', u'2,'), + (0x1F104, '3', u'3,'), + (0x1F105, '3', u'4,'), + (0x1F106, '3', u'5,'), + (0x1F107, '3', u'6,'), + (0x1F108, '3', u'7,'), + ] + +def _seg_72(): + return [ + (0x1F109, '3', u'8,'), + (0x1F10A, '3', u'9,'), + (0x1F10B, 'V'), + (0x1F110, '3', u'(a)'), + (0x1F111, '3', u'(b)'), + (0x1F112, '3', u'(c)'), + (0x1F113, '3', u'(d)'), + (0x1F114, '3', u'(e)'), + (0x1F115, '3', u'(f)'), + (0x1F116, '3', u'(g)'), + (0x1F117, '3', u'(h)'), + (0x1F118, '3', u'(i)'), + (0x1F119, '3', u'(j)'), + (0x1F11A, '3', u'(k)'), + (0x1F11B, '3', u'(l)'), + (0x1F11C, '3', u'(m)'), + (0x1F11D, '3', u'(n)'), + (0x1F11E, '3', u'(o)'), + (0x1F11F, '3', u'(p)'), + (0x1F120, '3', u'(q)'), + (0x1F121, '3', u'(r)'), + (0x1F122, '3', u'(s)'), + (0x1F123, '3', u'(t)'), + (0x1F124, '3', u'(u)'), + (0x1F125, '3', u'(v)'), + (0x1F126, '3', u'(w)'), + (0x1F127, '3', u'(x)'), + (0x1F128, '3', u'(y)'), + (0x1F129, '3', u'(z)'), + (0x1F12A, 'M', u'〔s〕'), + (0x1F12B, 'M', u'c'), + (0x1F12C, 'M', u'r'), + (0x1F12D, 'M', u'cd'), + (0x1F12E, 'M', u'wz'), + (0x1F12F, 'V'), + (0x1F130, 'M', u'a'), + (0x1F131, 'M', u'b'), + (0x1F132, 'M', u'c'), + (0x1F133, 'M', u'd'), + (0x1F134, 'M', u'e'), + (0x1F135, 'M', u'f'), + (0x1F136, 'M', u'g'), + (0x1F137, 'M', u'h'), + (0x1F138, 'M', u'i'), + (0x1F139, 'M', u'j'), + (0x1F13A, 'M', u'k'), + (0x1F13B, 'M', u'l'), + (0x1F13C, 'M', u'm'), + (0x1F13D, 'M', u'n'), + (0x1F13E, 'M', u'o'), + (0x1F13F, 'M', u'p'), + (0x1F140, 'M', u'q'), + (0x1F141, 'M', u'r'), + (0x1F142, 'M', u's'), + (0x1F143, 'M', u't'), + (0x1F144, 'M', u'u'), + (0x1F145, 'M', u'v'), + (0x1F146, 'M', u'w'), + (0x1F147, 'M', u'x'), + (0x1F148, 'M', u'y'), + (0x1F149, 'M', u'z'), + (0x1F14A, 'M', u'hv'), + (0x1F14B, 'M', u'mv'), + (0x1F14C, 'M', u'sd'), + (0x1F14D, 'M', u'ss'), + (0x1F14E, 'M', u'ppv'), + (0x1F14F, 'M', u'wc'), + (0x1F150, 'V'), + (0x1F16A, 'M', u'mc'), + (0x1F16B, 'M', u'md'), + (0x1F16C, 'M', u'mr'), + (0x1F16D, 'V'), + (0x1F190, 'M', u'dj'), + (0x1F191, 'V'), + (0x1F1AE, 'X'), + (0x1F1E6, 'V'), + (0x1F200, 'M', u'ほか'), + (0x1F201, 'M', u'ココ'), + (0x1F202, 'M', u'サ'), + (0x1F203, 'X'), + (0x1F210, 'M', u'手'), + (0x1F211, 'M', u'字'), + (0x1F212, 'M', u'双'), + (0x1F213, 'M', u'デ'), + (0x1F214, 'M', u'二'), + (0x1F215, 'M', u'多'), + (0x1F216, 'M', u'解'), + (0x1F217, 'M', u'天'), + (0x1F218, 'M', u'交'), + (0x1F219, 'M', u'映'), + (0x1F21A, 'M', u'無'), + (0x1F21B, 'M', u'料'), + (0x1F21C, 'M', u'前'), + (0x1F21D, 'M', u'後'), + (0x1F21E, 'M', u'再'), + (0x1F21F, 'M', u'新'), + (0x1F220, 'M', u'初'), + (0x1F221, 'M', u'終'), + (0x1F222, 'M', u'生'), + (0x1F223, 'M', u'販'), + ] + +def _seg_73(): + return [ + (0x1F224, 'M', u'声'), + (0x1F225, 'M', u'吹'), + (0x1F226, 'M', u'演'), + (0x1F227, 'M', u'投'), + (0x1F228, 'M', u'捕'), + (0x1F229, 'M', u'一'), + (0x1F22A, 'M', u'三'), + (0x1F22B, 'M', u'遊'), + (0x1F22C, 'M', u'左'), + (0x1F22D, 'M', u'中'), + (0x1F22E, 'M', u'右'), + (0x1F22F, 'M', u'指'), + (0x1F230, 'M', u'走'), + (0x1F231, 'M', u'打'), + (0x1F232, 'M', u'禁'), + (0x1F233, 'M', u'空'), + (0x1F234, 'M', u'合'), + (0x1F235, 'M', u'満'), + (0x1F236, 'M', u'有'), + (0x1F237, 'M', u'月'), + (0x1F238, 'M', u'申'), + (0x1F239, 'M', u'割'), + (0x1F23A, 'M', u'営'), + (0x1F23B, 'M', u'配'), + (0x1F23C, 'X'), + (0x1F240, 'M', u'〔本〕'), + (0x1F241, 'M', u'〔三〕'), + (0x1F242, 'M', u'〔二〕'), + (0x1F243, 'M', u'〔安〕'), + (0x1F244, 'M', u'〔点〕'), + (0x1F245, 'M', u'〔打〕'), + (0x1F246, 'M', u'〔盗〕'), + (0x1F247, 'M', u'〔勝〕'), + (0x1F248, 'M', u'〔敗〕'), + (0x1F249, 'X'), + (0x1F250, 'M', u'得'), + (0x1F251, 'M', u'可'), + (0x1F252, 'X'), + (0x1F260, 'V'), + (0x1F266, 'X'), + (0x1F300, 'V'), + (0x1F6D8, 'X'), + (0x1F6E0, 'V'), + (0x1F6ED, 'X'), + (0x1F6F0, 'V'), + (0x1F6FD, 'X'), + (0x1F700, 'V'), + (0x1F774, 'X'), + (0x1F780, 'V'), + (0x1F7D9, 'X'), + (0x1F7E0, 'V'), + (0x1F7EC, 'X'), + (0x1F800, 'V'), + (0x1F80C, 'X'), + (0x1F810, 'V'), + (0x1F848, 'X'), + (0x1F850, 'V'), + (0x1F85A, 'X'), + (0x1F860, 'V'), + (0x1F888, 'X'), + (0x1F890, 'V'), + (0x1F8AE, 'X'), + (0x1F8B0, 'V'), + (0x1F8B2, 'X'), + (0x1F900, 'V'), + (0x1F979, 'X'), + (0x1F97A, 'V'), + (0x1F9CC, 'X'), + (0x1F9CD, 'V'), + (0x1FA54, 'X'), + (0x1FA60, 'V'), + (0x1FA6E, 'X'), + (0x1FA70, 'V'), + (0x1FA75, 'X'), + (0x1FA78, 'V'), + (0x1FA7B, 'X'), + (0x1FA80, 'V'), + (0x1FA87, 'X'), + (0x1FA90, 'V'), + (0x1FAA9, 'X'), + (0x1FAB0, 'V'), + (0x1FAB7, 'X'), + (0x1FAC0, 'V'), + (0x1FAC3, 'X'), + (0x1FAD0, 'V'), + (0x1FAD7, 'X'), + (0x1FB00, 'V'), + (0x1FB93, 'X'), + (0x1FB94, 'V'), + (0x1FBCB, 'X'), + (0x1FBF0, 'M', u'0'), + (0x1FBF1, 'M', u'1'), + (0x1FBF2, 'M', u'2'), + (0x1FBF3, 'M', u'3'), + (0x1FBF4, 'M', u'4'), + (0x1FBF5, 'M', u'5'), + (0x1FBF6, 'M', u'6'), + (0x1FBF7, 'M', u'7'), + (0x1FBF8, 'M', u'8'), + (0x1FBF9, 'M', u'9'), + ] + +def _seg_74(): + return [ + (0x1FBFA, 'X'), + (0x20000, 'V'), + (0x2A6DE, 'X'), + (0x2A700, 'V'), + (0x2B735, 'X'), + (0x2B740, 'V'), + (0x2B81E, 'X'), + (0x2B820, 'V'), + (0x2CEA2, 'X'), + (0x2CEB0, 'V'), + (0x2EBE1, 'X'), + (0x2F800, 'M', u'丽'), + (0x2F801, 'M', u'丸'), + (0x2F802, 'M', u'乁'), + (0x2F803, 'M', u'𠄢'), + (0x2F804, 'M', u'你'), + (0x2F805, 'M', u'侮'), + (0x2F806, 'M', u'侻'), + (0x2F807, 'M', u'倂'), + (0x2F808, 'M', u'偺'), + (0x2F809, 'M', u'備'), + (0x2F80A, 'M', u'僧'), + (0x2F80B, 'M', u'像'), + (0x2F80C, 'M', u'㒞'), + (0x2F80D, 'M', u'𠘺'), + (0x2F80E, 'M', u'免'), + (0x2F80F, 'M', u'兔'), + (0x2F810, 'M', u'兤'), + (0x2F811, 'M', u'具'), + (0x2F812, 'M', u'𠔜'), + (0x2F813, 'M', u'㒹'), + (0x2F814, 'M', u'內'), + (0x2F815, 'M', u'再'), + (0x2F816, 'M', u'𠕋'), + (0x2F817, 'M', u'冗'), + (0x2F818, 'M', u'冤'), + (0x2F819, 'M', u'仌'), + (0x2F81A, 'M', u'冬'), + (0x2F81B, 'M', u'况'), + (0x2F81C, 'M', u'𩇟'), + (0x2F81D, 'M', u'凵'), + (0x2F81E, 'M', u'刃'), + (0x2F81F, 'M', u'㓟'), + (0x2F820, 'M', u'刻'), + (0x2F821, 'M', u'剆'), + (0x2F822, 'M', u'割'), + (0x2F823, 'M', u'剷'), + (0x2F824, 'M', u'㔕'), + (0x2F825, 'M', u'勇'), + (0x2F826, 'M', u'勉'), + (0x2F827, 'M', u'勤'), + (0x2F828, 'M', u'勺'), + (0x2F829, 'M', u'包'), + (0x2F82A, 'M', u'匆'), + (0x2F82B, 'M', u'北'), + (0x2F82C, 'M', u'卉'), + (0x2F82D, 'M', u'卑'), + (0x2F82E, 'M', u'博'), + (0x2F82F, 'M', u'即'), + (0x2F830, 'M', u'卽'), + (0x2F831, 'M', u'卿'), + (0x2F834, 'M', u'𠨬'), + (0x2F835, 'M', u'灰'), + (0x2F836, 'M', u'及'), + (0x2F837, 'M', u'叟'), + (0x2F838, 'M', u'𠭣'), + (0x2F839, 'M', u'叫'), + (0x2F83A, 'M', u'叱'), + (0x2F83B, 'M', u'吆'), + (0x2F83C, 'M', u'咞'), + (0x2F83D, 'M', u'吸'), + (0x2F83E, 'M', u'呈'), + (0x2F83F, 'M', u'周'), + (0x2F840, 'M', u'咢'), + (0x2F841, 'M', u'哶'), + (0x2F842, 'M', u'唐'), + (0x2F843, 'M', u'啓'), + (0x2F844, 'M', u'啣'), + (0x2F845, 'M', u'善'), + (0x2F847, 'M', u'喙'), + (0x2F848, 'M', u'喫'), + (0x2F849, 'M', u'喳'), + (0x2F84A, 'M', u'嗂'), + (0x2F84B, 'M', u'圖'), + (0x2F84C, 'M', u'嘆'), + (0x2F84D, 'M', u'圗'), + (0x2F84E, 'M', u'噑'), + (0x2F84F, 'M', u'噴'), + (0x2F850, 'M', u'切'), + (0x2F851, 'M', u'壮'), + (0x2F852, 'M', u'城'), + (0x2F853, 'M', u'埴'), + (0x2F854, 'M', u'堍'), + (0x2F855, 'M', u'型'), + (0x2F856, 'M', u'堲'), + (0x2F857, 'M', u'報'), + (0x2F858, 'M', u'墬'), + (0x2F859, 'M', u'𡓤'), + (0x2F85A, 'M', u'売'), + (0x2F85B, 'M', u'壷'), + ] + +def _seg_75(): + return [ + (0x2F85C, 'M', u'夆'), + (0x2F85D, 'M', u'多'), + (0x2F85E, 'M', u'夢'), + (0x2F85F, 'M', u'奢'), + (0x2F860, 'M', u'𡚨'), + (0x2F861, 'M', u'𡛪'), + (0x2F862, 'M', u'姬'), + (0x2F863, 'M', u'娛'), + (0x2F864, 'M', u'娧'), + (0x2F865, 'M', u'姘'), + (0x2F866, 'M', u'婦'), + (0x2F867, 'M', u'㛮'), + (0x2F868, 'X'), + (0x2F869, 'M', u'嬈'), + (0x2F86A, 'M', u'嬾'), + (0x2F86C, 'M', u'𡧈'), + (0x2F86D, 'M', u'寃'), + (0x2F86E, 'M', u'寘'), + (0x2F86F, 'M', u'寧'), + (0x2F870, 'M', u'寳'), + (0x2F871, 'M', u'𡬘'), + (0x2F872, 'M', u'寿'), + (0x2F873, 'M', u'将'), + (0x2F874, 'X'), + (0x2F875, 'M', u'尢'), + (0x2F876, 'M', u'㞁'), + (0x2F877, 'M', u'屠'), + (0x2F878, 'M', u'屮'), + (0x2F879, 'M', u'峀'), + (0x2F87A, 'M', u'岍'), + (0x2F87B, 'M', u'𡷤'), + (0x2F87C, 'M', u'嵃'), + (0x2F87D, 'M', u'𡷦'), + (0x2F87E, 'M', u'嵮'), + (0x2F87F, 'M', u'嵫'), + (0x2F880, 'M', u'嵼'), + (0x2F881, 'M', u'巡'), + (0x2F882, 'M', u'巢'), + (0x2F883, 'M', u'㠯'), + (0x2F884, 'M', u'巽'), + (0x2F885, 'M', u'帨'), + (0x2F886, 'M', u'帽'), + (0x2F887, 'M', u'幩'), + (0x2F888, 'M', u'㡢'), + (0x2F889, 'M', u'𢆃'), + (0x2F88A, 'M', u'㡼'), + (0x2F88B, 'M', u'庰'), + (0x2F88C, 'M', u'庳'), + (0x2F88D, 'M', u'庶'), + (0x2F88E, 'M', u'廊'), + (0x2F88F, 'M', u'𪎒'), + (0x2F890, 'M', u'廾'), + (0x2F891, 'M', u'𢌱'), + (0x2F893, 'M', u'舁'), + (0x2F894, 'M', u'弢'), + (0x2F896, 'M', u'㣇'), + (0x2F897, 'M', u'𣊸'), + (0x2F898, 'M', u'𦇚'), + (0x2F899, 'M', u'形'), + (0x2F89A, 'M', u'彫'), + (0x2F89B, 'M', u'㣣'), + (0x2F89C, 'M', u'徚'), + (0x2F89D, 'M', u'忍'), + (0x2F89E, 'M', u'志'), + (0x2F89F, 'M', u'忹'), + (0x2F8A0, 'M', u'悁'), + (0x2F8A1, 'M', u'㤺'), + (0x2F8A2, 'M', u'㤜'), + (0x2F8A3, 'M', u'悔'), + (0x2F8A4, 'M', u'𢛔'), + (0x2F8A5, 'M', u'惇'), + (0x2F8A6, 'M', u'慈'), + (0x2F8A7, 'M', u'慌'), + (0x2F8A8, 'M', u'慎'), + (0x2F8A9, 'M', u'慌'), + (0x2F8AA, 'M', u'慺'), + (0x2F8AB, 'M', u'憎'), + (0x2F8AC, 'M', u'憲'), + (0x2F8AD, 'M', u'憤'), + (0x2F8AE, 'M', u'憯'), + (0x2F8AF, 'M', u'懞'), + (0x2F8B0, 'M', u'懲'), + (0x2F8B1, 'M', u'懶'), + (0x2F8B2, 'M', u'成'), + (0x2F8B3, 'M', u'戛'), + (0x2F8B4, 'M', u'扝'), + (0x2F8B5, 'M', u'抱'), + (0x2F8B6, 'M', u'拔'), + (0x2F8B7, 'M', u'捐'), + (0x2F8B8, 'M', u'𢬌'), + (0x2F8B9, 'M', u'挽'), + (0x2F8BA, 'M', u'拼'), + (0x2F8BB, 'M', u'捨'), + (0x2F8BC, 'M', u'掃'), + (0x2F8BD, 'M', u'揤'), + (0x2F8BE, 'M', u'𢯱'), + (0x2F8BF, 'M', u'搢'), + (0x2F8C0, 'M', u'揅'), + (0x2F8C1, 'M', u'掩'), + (0x2F8C2, 'M', u'㨮'), + ] + +def _seg_76(): + return [ + (0x2F8C3, 'M', u'摩'), + (0x2F8C4, 'M', u'摾'), + (0x2F8C5, 'M', u'撝'), + (0x2F8C6, 'M', u'摷'), + (0x2F8C7, 'M', u'㩬'), + (0x2F8C8, 'M', u'敏'), + (0x2F8C9, 'M', u'敬'), + (0x2F8CA, 'M', u'𣀊'), + (0x2F8CB, 'M', u'旣'), + (0x2F8CC, 'M', u'書'), + (0x2F8CD, 'M', u'晉'), + (0x2F8CE, 'M', u'㬙'), + (0x2F8CF, 'M', u'暑'), + (0x2F8D0, 'M', u'㬈'), + (0x2F8D1, 'M', u'㫤'), + (0x2F8D2, 'M', u'冒'), + (0x2F8D3, 'M', u'冕'), + (0x2F8D4, 'M', u'最'), + (0x2F8D5, 'M', u'暜'), + (0x2F8D6, 'M', u'肭'), + (0x2F8D7, 'M', u'䏙'), + (0x2F8D8, 'M', u'朗'), + (0x2F8D9, 'M', u'望'), + (0x2F8DA, 'M', u'朡'), + (0x2F8DB, 'M', u'杞'), + (0x2F8DC, 'M', u'杓'), + (0x2F8DD, 'M', u'𣏃'), + (0x2F8DE, 'M', u'㭉'), + (0x2F8DF, 'M', u'柺'), + (0x2F8E0, 'M', u'枅'), + (0x2F8E1, 'M', u'桒'), + (0x2F8E2, 'M', u'梅'), + (0x2F8E3, 'M', u'𣑭'), + (0x2F8E4, 'M', u'梎'), + (0x2F8E5, 'M', u'栟'), + (0x2F8E6, 'M', u'椔'), + (0x2F8E7, 'M', u'㮝'), + (0x2F8E8, 'M', u'楂'), + (0x2F8E9, 'M', u'榣'), + (0x2F8EA, 'M', u'槪'), + (0x2F8EB, 'M', u'檨'), + (0x2F8EC, 'M', u'𣚣'), + (0x2F8ED, 'M', u'櫛'), + (0x2F8EE, 'M', u'㰘'), + (0x2F8EF, 'M', u'次'), + (0x2F8F0, 'M', u'𣢧'), + (0x2F8F1, 'M', u'歔'), + (0x2F8F2, 'M', u'㱎'), + (0x2F8F3, 'M', u'歲'), + (0x2F8F4, 'M', u'殟'), + (0x2F8F5, 'M', u'殺'), + (0x2F8F6, 'M', u'殻'), + (0x2F8F7, 'M', u'𣪍'), + (0x2F8F8, 'M', u'𡴋'), + (0x2F8F9, 'M', u'𣫺'), + (0x2F8FA, 'M', u'汎'), + (0x2F8FB, 'M', u'𣲼'), + (0x2F8FC, 'M', u'沿'), + (0x2F8FD, 'M', u'泍'), + (0x2F8FE, 'M', u'汧'), + (0x2F8FF, 'M', u'洖'), + (0x2F900, 'M', u'派'), + (0x2F901, 'M', u'海'), + (0x2F902, 'M', u'流'), + (0x2F903, 'M', u'浩'), + (0x2F904, 'M', u'浸'), + (0x2F905, 'M', u'涅'), + (0x2F906, 'M', u'𣴞'), + (0x2F907, 'M', u'洴'), + (0x2F908, 'M', u'港'), + (0x2F909, 'M', u'湮'), + (0x2F90A, 'M', u'㴳'), + (0x2F90B, 'M', u'滋'), + (0x2F90C, 'M', u'滇'), + (0x2F90D, 'M', u'𣻑'), + (0x2F90E, 'M', u'淹'), + (0x2F90F, 'M', u'潮'), + (0x2F910, 'M', u'𣽞'), + (0x2F911, 'M', u'𣾎'), + (0x2F912, 'M', u'濆'), + (0x2F913, 'M', u'瀹'), + (0x2F914, 'M', u'瀞'), + (0x2F915, 'M', u'瀛'), + (0x2F916, 'M', u'㶖'), + (0x2F917, 'M', u'灊'), + (0x2F918, 'M', u'災'), + (0x2F919, 'M', u'灷'), + (0x2F91A, 'M', u'炭'), + (0x2F91B, 'M', u'𠔥'), + (0x2F91C, 'M', u'煅'), + (0x2F91D, 'M', u'𤉣'), + (0x2F91E, 'M', u'熜'), + (0x2F91F, 'X'), + (0x2F920, 'M', u'爨'), + (0x2F921, 'M', u'爵'), + (0x2F922, 'M', u'牐'), + (0x2F923, 'M', u'𤘈'), + (0x2F924, 'M', u'犀'), + (0x2F925, 'M', u'犕'), + (0x2F926, 'M', u'𤜵'), + ] + +def _seg_77(): + return [ + (0x2F927, 'M', u'𤠔'), + (0x2F928, 'M', u'獺'), + (0x2F929, 'M', u'王'), + (0x2F92A, 'M', u'㺬'), + (0x2F92B, 'M', u'玥'), + (0x2F92C, 'M', u'㺸'), + (0x2F92E, 'M', u'瑇'), + (0x2F92F, 'M', u'瑜'), + (0x2F930, 'M', u'瑱'), + (0x2F931, 'M', u'璅'), + (0x2F932, 'M', u'瓊'), + (0x2F933, 'M', u'㼛'), + (0x2F934, 'M', u'甤'), + (0x2F935, 'M', u'𤰶'), + (0x2F936, 'M', u'甾'), + (0x2F937, 'M', u'𤲒'), + (0x2F938, 'M', u'異'), + (0x2F939, 'M', u'𢆟'), + (0x2F93A, 'M', u'瘐'), + (0x2F93B, 'M', u'𤾡'), + (0x2F93C, 'M', u'𤾸'), + (0x2F93D, 'M', u'𥁄'), + (0x2F93E, 'M', u'㿼'), + (0x2F93F, 'M', u'䀈'), + (0x2F940, 'M', u'直'), + (0x2F941, 'M', u'𥃳'), + (0x2F942, 'M', u'𥃲'), + (0x2F943, 'M', u'𥄙'), + (0x2F944, 'M', u'𥄳'), + (0x2F945, 'M', u'眞'), + (0x2F946, 'M', u'真'), + (0x2F948, 'M', u'睊'), + (0x2F949, 'M', u'䀹'), + (0x2F94A, 'M', u'瞋'), + (0x2F94B, 'M', u'䁆'), + (0x2F94C, 'M', u'䂖'), + (0x2F94D, 'M', u'𥐝'), + (0x2F94E, 'M', u'硎'), + (0x2F94F, 'M', u'碌'), + (0x2F950, 'M', u'磌'), + (0x2F951, 'M', u'䃣'), + (0x2F952, 'M', u'𥘦'), + (0x2F953, 'M', u'祖'), + (0x2F954, 'M', u'𥚚'), + (0x2F955, 'M', u'𥛅'), + (0x2F956, 'M', u'福'), + (0x2F957, 'M', u'秫'), + (0x2F958, 'M', u'䄯'), + (0x2F959, 'M', u'穀'), + (0x2F95A, 'M', u'穊'), + (0x2F95B, 'M', u'穏'), + (0x2F95C, 'M', u'𥥼'), + (0x2F95D, 'M', u'𥪧'), + (0x2F95F, 'X'), + (0x2F960, 'M', u'䈂'), + (0x2F961, 'M', u'𥮫'), + (0x2F962, 'M', u'篆'), + (0x2F963, 'M', u'築'), + (0x2F964, 'M', u'䈧'), + (0x2F965, 'M', u'𥲀'), + (0x2F966, 'M', u'糒'), + (0x2F967, 'M', u'䊠'), + (0x2F968, 'M', u'糨'), + (0x2F969, 'M', u'糣'), + (0x2F96A, 'M', u'紀'), + (0x2F96B, 'M', u'𥾆'), + (0x2F96C, 'M', u'絣'), + (0x2F96D, 'M', u'䌁'), + (0x2F96E, 'M', u'緇'), + (0x2F96F, 'M', u'縂'), + (0x2F970, 'M', u'繅'), + (0x2F971, 'M', u'䌴'), + (0x2F972, 'M', u'𦈨'), + (0x2F973, 'M', u'𦉇'), + (0x2F974, 'M', u'䍙'), + (0x2F975, 'M', u'𦋙'), + (0x2F976, 'M', u'罺'), + (0x2F977, 'M', u'𦌾'), + (0x2F978, 'M', u'羕'), + (0x2F979, 'M', u'翺'), + (0x2F97A, 'M', u'者'), + (0x2F97B, 'M', u'𦓚'), + (0x2F97C, 'M', u'𦔣'), + (0x2F97D, 'M', u'聠'), + (0x2F97E, 'M', u'𦖨'), + (0x2F97F, 'M', u'聰'), + (0x2F980, 'M', u'𣍟'), + (0x2F981, 'M', u'䏕'), + (0x2F982, 'M', u'育'), + (0x2F983, 'M', u'脃'), + (0x2F984, 'M', u'䐋'), + (0x2F985, 'M', u'脾'), + (0x2F986, 'M', u'媵'), + (0x2F987, 'M', u'𦞧'), + (0x2F988, 'M', u'𦞵'), + (0x2F989, 'M', u'𣎓'), + (0x2F98A, 'M', u'𣎜'), + (0x2F98B, 'M', u'舁'), + (0x2F98C, 'M', u'舄'), + (0x2F98D, 'M', u'辞'), + ] + +def _seg_78(): + return [ + (0x2F98E, 'M', u'䑫'), + (0x2F98F, 'M', u'芑'), + (0x2F990, 'M', u'芋'), + (0x2F991, 'M', u'芝'), + (0x2F992, 'M', u'劳'), + (0x2F993, 'M', u'花'), + (0x2F994, 'M', u'芳'), + (0x2F995, 'M', u'芽'), + (0x2F996, 'M', u'苦'), + (0x2F997, 'M', u'𦬼'), + (0x2F998, 'M', u'若'), + (0x2F999, 'M', u'茝'), + (0x2F99A, 'M', u'荣'), + (0x2F99B, 'M', u'莭'), + (0x2F99C, 'M', u'茣'), + (0x2F99D, 'M', u'莽'), + (0x2F99E, 'M', u'菧'), + (0x2F99F, 'M', u'著'), + (0x2F9A0, 'M', u'荓'), + (0x2F9A1, 'M', u'菊'), + (0x2F9A2, 'M', u'菌'), + (0x2F9A3, 'M', u'菜'), + (0x2F9A4, 'M', u'𦰶'), + (0x2F9A5, 'M', u'𦵫'), + (0x2F9A6, 'M', u'𦳕'), + (0x2F9A7, 'M', u'䔫'), + (0x2F9A8, 'M', u'蓱'), + (0x2F9A9, 'M', u'蓳'), + (0x2F9AA, 'M', u'蔖'), + (0x2F9AB, 'M', u'𧏊'), + (0x2F9AC, 'M', u'蕤'), + (0x2F9AD, 'M', u'𦼬'), + (0x2F9AE, 'M', u'䕝'), + (0x2F9AF, 'M', u'䕡'), + (0x2F9B0, 'M', u'𦾱'), + (0x2F9B1, 'M', u'𧃒'), + (0x2F9B2, 'M', u'䕫'), + (0x2F9B3, 'M', u'虐'), + (0x2F9B4, 'M', u'虜'), + (0x2F9B5, 'M', u'虧'), + (0x2F9B6, 'M', u'虩'), + (0x2F9B7, 'M', u'蚩'), + (0x2F9B8, 'M', u'蚈'), + (0x2F9B9, 'M', u'蜎'), + (0x2F9BA, 'M', u'蛢'), + (0x2F9BB, 'M', u'蝹'), + (0x2F9BC, 'M', u'蜨'), + (0x2F9BD, 'M', u'蝫'), + (0x2F9BE, 'M', u'螆'), + (0x2F9BF, 'X'), + (0x2F9C0, 'M', u'蟡'), + (0x2F9C1, 'M', u'蠁'), + (0x2F9C2, 'M', u'䗹'), + (0x2F9C3, 'M', u'衠'), + (0x2F9C4, 'M', u'衣'), + (0x2F9C5, 'M', u'𧙧'), + (0x2F9C6, 'M', u'裗'), + (0x2F9C7, 'M', u'裞'), + (0x2F9C8, 'M', u'䘵'), + (0x2F9C9, 'M', u'裺'), + (0x2F9CA, 'M', u'㒻'), + (0x2F9CB, 'M', u'𧢮'), + (0x2F9CC, 'M', u'𧥦'), + (0x2F9CD, 'M', u'䚾'), + (0x2F9CE, 'M', u'䛇'), + (0x2F9CF, 'M', u'誠'), + (0x2F9D0, 'M', u'諭'), + (0x2F9D1, 'M', u'變'), + (0x2F9D2, 'M', u'豕'), + (0x2F9D3, 'M', u'𧲨'), + (0x2F9D4, 'M', u'貫'), + (0x2F9D5, 'M', u'賁'), + (0x2F9D6, 'M', u'贛'), + (0x2F9D7, 'M', u'起'), + (0x2F9D8, 'M', u'𧼯'), + (0x2F9D9, 'M', u'𠠄'), + (0x2F9DA, 'M', u'跋'), + (0x2F9DB, 'M', u'趼'), + (0x2F9DC, 'M', u'跰'), + (0x2F9DD, 'M', u'𠣞'), + (0x2F9DE, 'M', u'軔'), + (0x2F9DF, 'M', u'輸'), + (0x2F9E0, 'M', u'𨗒'), + (0x2F9E1, 'M', u'𨗭'), + (0x2F9E2, 'M', u'邔'), + (0x2F9E3, 'M', u'郱'), + (0x2F9E4, 'M', u'鄑'), + (0x2F9E5, 'M', u'𨜮'), + (0x2F9E6, 'M', u'鄛'), + (0x2F9E7, 'M', u'鈸'), + (0x2F9E8, 'M', u'鋗'), + (0x2F9E9, 'M', u'鋘'), + (0x2F9EA, 'M', u'鉼'), + (0x2F9EB, 'M', u'鏹'), + (0x2F9EC, 'M', u'鐕'), + (0x2F9ED, 'M', u'𨯺'), + (0x2F9EE, 'M', u'開'), + (0x2F9EF, 'M', u'䦕'), + (0x2F9F0, 'M', u'閷'), + (0x2F9F1, 'M', u'𨵷'), + ] + +def _seg_79(): + return [ + (0x2F9F2, 'M', u'䧦'), + (0x2F9F3, 'M', u'雃'), + (0x2F9F4, 'M', u'嶲'), + (0x2F9F5, 'M', u'霣'), + (0x2F9F6, 'M', u'𩅅'), + (0x2F9F7, 'M', u'𩈚'), + (0x2F9F8, 'M', u'䩮'), + (0x2F9F9, 'M', u'䩶'), + (0x2F9FA, 'M', u'韠'), + (0x2F9FB, 'M', u'𩐊'), + (0x2F9FC, 'M', u'䪲'), + (0x2F9FD, 'M', u'𩒖'), + (0x2F9FE, 'M', u'頋'), + (0x2FA00, 'M', u'頩'), + (0x2FA01, 'M', u'𩖶'), + (0x2FA02, 'M', u'飢'), + (0x2FA03, 'M', u'䬳'), + (0x2FA04, 'M', u'餩'), + (0x2FA05, 'M', u'馧'), + (0x2FA06, 'M', u'駂'), + (0x2FA07, 'M', u'駾'), + (0x2FA08, 'M', u'䯎'), + (0x2FA09, 'M', u'𩬰'), + (0x2FA0A, 'M', u'鬒'), + (0x2FA0B, 'M', u'鱀'), + (0x2FA0C, 'M', u'鳽'), + (0x2FA0D, 'M', u'䳎'), + (0x2FA0E, 'M', u'䳭'), + (0x2FA0F, 'M', u'鵧'), + (0x2FA10, 'M', u'𪃎'), + (0x2FA11, 'M', u'䳸'), + (0x2FA12, 'M', u'𪄅'), + (0x2FA13, 'M', u'𪈎'), + (0x2FA14, 'M', u'𪊑'), + (0x2FA15, 'M', u'麻'), + (0x2FA16, 'M', u'䵖'), + (0x2FA17, 'M', u'黹'), + (0x2FA18, 'M', u'黾'), + (0x2FA19, 'M', u'鼅'), + (0x2FA1A, 'M', u'鼏'), + (0x2FA1B, 'M', u'鼖'), + (0x2FA1C, 'M', u'鼻'), + (0x2FA1D, 'M', u'𪘀'), + (0x2FA1E, 'X'), + (0x30000, 'V'), + (0x3134B, 'X'), + (0xE0100, 'I'), + (0xE01F0, 'X'), + ] + +uts46data = tuple( + _seg_0() + + _seg_1() + + _seg_2() + + _seg_3() + + _seg_4() + + _seg_5() + + _seg_6() + + _seg_7() + + _seg_8() + + _seg_9() + + _seg_10() + + _seg_11() + + _seg_12() + + _seg_13() + + _seg_14() + + _seg_15() + + _seg_16() + + _seg_17() + + _seg_18() + + _seg_19() + + _seg_20() + + _seg_21() + + _seg_22() + + _seg_23() + + _seg_24() + + _seg_25() + + _seg_26() + + _seg_27() + + _seg_28() + + _seg_29() + + _seg_30() + + _seg_31() + + _seg_32() + + _seg_33() + + _seg_34() + + _seg_35() + + _seg_36() + + _seg_37() + + _seg_38() + + _seg_39() + + _seg_40() + + _seg_41() + + _seg_42() + + _seg_43() + + _seg_44() + + _seg_45() + + _seg_46() + + _seg_47() + + _seg_48() + + _seg_49() + + _seg_50() + + _seg_51() + + _seg_52() + + _seg_53() + + _seg_54() + + _seg_55() + + _seg_56() + + _seg_57() + + _seg_58() + + _seg_59() + + _seg_60() + + _seg_61() + + _seg_62() + + _seg_63() + + _seg_64() + + _seg_65() + + _seg_66() + + _seg_67() + + _seg_68() + + _seg_69() + + _seg_70() + + _seg_71() + + _seg_72() + + _seg_73() + + _seg_74() + + _seg_75() + + _seg_76() + + _seg_77() + + _seg_78() + + _seg_79() +) diff --git a/WebCrawler/venv/Lib/site-packages/incremental-17.5.0.dist-info/DESCRIPTION.rst b/WebCrawler/venv/Lib/site-packages/incremental-17.5.0.dist-info/DESCRIPTION.rst new file mode 100644 index 0000000..e9341c8 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/incremental-17.5.0.dist-info/DESCRIPTION.rst @@ -0,0 +1,110 @@ +Incremental +=========== + +|travis| +|pypi| +|coverage| + +Incremental is a small library that versions your Python projects. + +API documentation can be found `here `_. + + +Quick Start +----------- + +Add this to your ``setup.py``\ 's ``setup()`` call, removing any other versioning arguments: + +.. code:: + + setup( + use_incremental=True, + setup_requires=['incremental'], + install_requires=['incremental'], # along with any other install dependencies + ... + } + + +Then run ``python -m incremental.update --create`` (you will need ``click`` installed from PyPI). +It will create a file in your package named ``_version.py`` and look like this: + +.. code:: + + from incremental import Version + + __version__ = Version("widgetbox", 17, 1, 0) + __all__ = ["__version__"] + + +Then, so users of your project can find your version, in your root package's ``__init__.py`` add: + +.. code:: + + from ._version import __version__ + + +Subsequent installations of your project will then use Incremental for versioning. + + +Incremental Versions +-------------------- + +``incremental.Version`` is a class that represents a version of a given project. +It is made up of the following elements (which are given during instantiation): + +- ``package`` (required), the name of the package this ``Version`` represents. +- ``major``, ``minor``, ``micro`` (all required), the X.Y.Z of your project's ``Version``. +- ``release_candidate`` (optional), set to 0 or higher to mark this ``Version`` being of a release candidate (also sometimes called a "prerelease"). +- ``dev`` (optional), set to 0 or higher to mark this ``Version`` as a development release. + +You can extract a PEP-440 compatible version string by using the following methods: + +- ``.local()``, which returns a ``str`` containing the full version plus any Git or SVN information, if available. An example output would be ``"17.1.1rc1+r123"`` or ``"3.7.0+rb2e812003b5d5fcf08efd1dffed6afa98d44ac8c"``. +- ``.public()``, which returns a ``str`` containing the full version, without any Git or SVN information. This is the version you should provide to users, or publicly use. An example output would be ``"13.2.0"``, ``"17.1.2dev1"``, or ``"18.8.0rc2"``. + +Calling ``repr()`` with a ``Version`` will give a Python-source-code representation of it, and calling ``str()`` with a ``Version`` will provide a string similar to ``'[Incremental, version 16.10.1]'``. + + +Updating +-------- + +Incremental includes a tool to automate updating your Incremental-using project's version called ``incremental.update``. +It updates the ``_version.py`` file and automatically updates some uses of Incremental versions from an indeterminate version to the current one. +It requires ``click`` from PyPI. + +``python -m incremental.update `` will perform updates on that package. +The commands that can be given after that will determine what the next version is. + +- ``--newversion=``, to set the project version to a fully-specified version (like 1.2.3, or 17.1.0dev1). +- ``--rc``, to set the project version to ``..0rc1`` if the current version is not a release candidate, or bump the release candidate number by 1 if it is. +- ``--dev``, to set the project development release number to 0 if it is not a development release, or bump the development release number by 1 if it is. +- ``--patch``, to increment the patch number of the release. This will also reset the release candidate number, pass ``--rc`` at the same time to increment the patch number and make it a release candidate. + +If you give no arguments, it will strip the release candidate number, making it a "full release". + +Incremental supports "indeterminate" versions, as a stand-in for the next "full" version. This can be used when the version which will be displayed to the end-user is unknown (for example "introduced in" or "deprecated in"). Incremental supports the following indeterminate versions: + +- ``Version("", "NEXT", 0, 0)`` +- `` NEXT`` + +When you run ``python -m incremental.update --rc``, these will be updated to real versions (assuming the target final version is 17.1.0): + +- ``Version("", 17, 1, 0, release_candidate=1)`` +- `` 17.1.0rc1`` + +Once the final version is made, it will become: + +- ``Version("", 17, 1, 0)`` +- `` 17.1.0`` + + +.. |coverage| image:: https://codecov.io/github/hawkowl/incremental/coverage.svg?branch=master +.. _coverage: https://codecov.io/github/hawkowl/incremental + +.. |travis| image:: https://travis-ci.org/hawkowl/incremental.svg?branch=master +.. _travis: http://travis-ci.org/hawkowl/incremental + +.. |pypi| image:: http://img.shields.io/pypi/v/incremental.svg +.. _pypi: https://pypi.python.org/pypi/incremental + + diff --git a/WebCrawler/venv/Lib/site-packages/incremental-17.5.0.dist-info/INSTALLER b/WebCrawler/venv/Lib/site-packages/incremental-17.5.0.dist-info/INSTALLER new file mode 100644 index 0000000..a1b589e --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/incremental-17.5.0.dist-info/INSTALLER @@ -0,0 +1 @@ +pip diff --git a/WebCrawler/venv/Lib/site-packages/incremental-17.5.0.dist-info/METADATA b/WebCrawler/venv/Lib/site-packages/incremental-17.5.0.dist-info/METADATA new file mode 100644 index 0000000..a37730a --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/incremental-17.5.0.dist-info/METADATA @@ -0,0 +1,133 @@ +Metadata-Version: 2.0 +Name: incremental +Version: 17.5.0 +Summary: UNKNOWN +Home-page: https://github.com/twisted/incremental +Author: Amber Brown +Author-email: hawkowl@twistedmatrix.com +License: MIT +Platform: UNKNOWN +Classifier: Intended Audience :: Developers +Classifier: License :: OSI Approved :: MIT License +Classifier: Programming Language :: Python :: 2 +Classifier: Programming Language :: Python :: 2.6 +Classifier: Programming Language :: Python :: 2.7 +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3.3 +Classifier: Programming Language :: Python :: 3.4 +Classifier: Programming Language :: Python :: 3.5 +Classifier: Programming Language :: Python :: 3.6 +Provides-Extra: scripts +Requires-Dist: click (>=6.0); extra == 'scripts' +Requires-Dist: twisted (>=16.4.0); extra == 'scripts' + +Incremental +=========== + +|travis| +|pypi| +|coverage| + +Incremental is a small library that versions your Python projects. + +API documentation can be found `here `_. + + +Quick Start +----------- + +Add this to your ``setup.py``\ 's ``setup()`` call, removing any other versioning arguments: + +.. code:: + + setup( + use_incremental=True, + setup_requires=['incremental'], + install_requires=['incremental'], # along with any other install dependencies + ... + } + + +Then run ``python -m incremental.update --create`` (you will need ``click`` installed from PyPI). +It will create a file in your package named ``_version.py`` and look like this: + +.. code:: + + from incremental import Version + + __version__ = Version("widgetbox", 17, 1, 0) + __all__ = ["__version__"] + + +Then, so users of your project can find your version, in your root package's ``__init__.py`` add: + +.. code:: + + from ._version import __version__ + + +Subsequent installations of your project will then use Incremental for versioning. + + +Incremental Versions +-------------------- + +``incremental.Version`` is a class that represents a version of a given project. +It is made up of the following elements (which are given during instantiation): + +- ``package`` (required), the name of the package this ``Version`` represents. +- ``major``, ``minor``, ``micro`` (all required), the X.Y.Z of your project's ``Version``. +- ``release_candidate`` (optional), set to 0 or higher to mark this ``Version`` being of a release candidate (also sometimes called a "prerelease"). +- ``dev`` (optional), set to 0 or higher to mark this ``Version`` as a development release. + +You can extract a PEP-440 compatible version string by using the following methods: + +- ``.local()``, which returns a ``str`` containing the full version plus any Git or SVN information, if available. An example output would be ``"17.1.1rc1+r123"`` or ``"3.7.0+rb2e812003b5d5fcf08efd1dffed6afa98d44ac8c"``. +- ``.public()``, which returns a ``str`` containing the full version, without any Git or SVN information. This is the version you should provide to users, or publicly use. An example output would be ``"13.2.0"``, ``"17.1.2dev1"``, or ``"18.8.0rc2"``. + +Calling ``repr()`` with a ``Version`` will give a Python-source-code representation of it, and calling ``str()`` with a ``Version`` will provide a string similar to ``'[Incremental, version 16.10.1]'``. + + +Updating +-------- + +Incremental includes a tool to automate updating your Incremental-using project's version called ``incremental.update``. +It updates the ``_version.py`` file and automatically updates some uses of Incremental versions from an indeterminate version to the current one. +It requires ``click`` from PyPI. + +``python -m incremental.update `` will perform updates on that package. +The commands that can be given after that will determine what the next version is. + +- ``--newversion=``, to set the project version to a fully-specified version (like 1.2.3, or 17.1.0dev1). +- ``--rc``, to set the project version to ``..0rc1`` if the current version is not a release candidate, or bump the release candidate number by 1 if it is. +- ``--dev``, to set the project development release number to 0 if it is not a development release, or bump the development release number by 1 if it is. +- ``--patch``, to increment the patch number of the release. This will also reset the release candidate number, pass ``--rc`` at the same time to increment the patch number and make it a release candidate. + +If you give no arguments, it will strip the release candidate number, making it a "full release". + +Incremental supports "indeterminate" versions, as a stand-in for the next "full" version. This can be used when the version which will be displayed to the end-user is unknown (for example "introduced in" or "deprecated in"). Incremental supports the following indeterminate versions: + +- ``Version("", "NEXT", 0, 0)`` +- `` NEXT`` + +When you run ``python -m incremental.update --rc``, these will be updated to real versions (assuming the target final version is 17.1.0): + +- ``Version("", 17, 1, 0, release_candidate=1)`` +- `` 17.1.0rc1`` + +Once the final version is made, it will become: + +- ``Version("", 17, 1, 0)`` +- `` 17.1.0`` + + +.. |coverage| image:: https://codecov.io/github/hawkowl/incremental/coverage.svg?branch=master +.. _coverage: https://codecov.io/github/hawkowl/incremental + +.. |travis| image:: https://travis-ci.org/hawkowl/incremental.svg?branch=master +.. _travis: http://travis-ci.org/hawkowl/incremental + +.. |pypi| image:: http://img.shields.io/pypi/v/incremental.svg +.. _pypi: https://pypi.python.org/pypi/incremental + + diff --git a/WebCrawler/venv/Lib/site-packages/incremental-17.5.0.dist-info/RECORD b/WebCrawler/venv/Lib/site-packages/incremental-17.5.0.dist-info/RECORD new file mode 100644 index 0000000..80c503b --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/incremental-17.5.0.dist-info/RECORD @@ -0,0 +1,24 @@ +exampleproj/__init__.py,sha256=wVUVOn7507BCBqkRvFS9HK53UcUjXzC5msC6bJRhS6s,229 +exampleproj/__pycache__/__init__.cpython-39.pyc,, +exampleproj/__pycache__/_version.cpython-39.pyc,, +exampleproj/_version.py,sha256=rg3bI25RROVfWYekpRHyamIjC57sP9ccNh-WT810ldo,220 +incremental-17.5.0.dist-info/DESCRIPTION.rst,sha256=CEhuO8j_yz24FpoelRI9wsJ--6kUA9hIksHUdhrgqLY,4818 +incremental-17.5.0.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4 +incremental-17.5.0.dist-info/METADATA,sha256=jxH6e70AY99WYfgmo1BM9ovHcuIv3411Gyw8bqsZKyk,5651 +incremental-17.5.0.dist-info/RECORD,, +incremental-17.5.0.dist-info/WHEEL,sha256=o2k-Qa-RMNIJmUdIc7KU6VWR_ErNRbWNlxDIpl7lm34,110 +incremental-17.5.0.dist-info/entry_points.txt,sha256=8Ap0rJ8G9THgI-ZYxAsGJVIffcEgM5y0GlhwZlLD0zM,83 +incremental-17.5.0.dist-info/metadata.json,sha256=kyDlobxp9BacDNwHCDiuuJzgskzntxABnKvt_53VXkQ,1032 +incremental-17.5.0.dist-info/top_level.txt,sha256=dVpYwi6ag8TlocBDEAfD7HydlIWoVmh6ESMv04izUdc,12 +incremental/__init__.py,sha256=F9sMkt4wNU0o9LilRL_bjh2x2UL3j0o8TQ0Azvt6A2E,9213 +incremental/__pycache__/__init__.cpython-39.pyc,, +incremental/__pycache__/_version.cpython-39.pyc,, +incremental/__pycache__/update.cpython-39.pyc,, +incremental/_version.py,sha256=VNbdIBHWnhj2KgEJU1jfCaXE0_TkOofTZlR9Jnnkc7Y,272 +incremental/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +incremental/tests/__pycache__/__init__.cpython-39.pyc,, +incremental/tests/__pycache__/test_update.cpython-39.pyc,, +incremental/tests/__pycache__/test_version.cpython-39.pyc,, +incremental/tests/test_update.py,sha256=Blsbg_4Yddbz12GJ7P1aZepXI0lLYC5mpibjkgeIQ0U,23630 +incremental/tests/test_version.py,sha256=zf6SO2T4fLOvmjEKIQWwismMDQoCygqoTnb1Iscn18o,14700 +incremental/update.py,sha256=XdWEC3ObYusl1XnrQhwsDzxCeQDjHTlnrqYWYpfGn18,6150 diff --git a/WebCrawler/venv/Lib/site-packages/incremental-17.5.0.dist-info/WHEEL b/WebCrawler/venv/Lib/site-packages/incremental-17.5.0.dist-info/WHEEL new file mode 100644 index 0000000..8b6dd1b --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/incremental-17.5.0.dist-info/WHEEL @@ -0,0 +1,6 @@ +Wheel-Version: 1.0 +Generator: bdist_wheel (0.29.0) +Root-Is-Purelib: true +Tag: py2-none-any +Tag: py3-none-any + diff --git a/WebCrawler/venv/Lib/site-packages/incremental-17.5.0.dist-info/entry_points.txt b/WebCrawler/venv/Lib/site-packages/incremental-17.5.0.dist-info/entry_points.txt new file mode 100644 index 0000000..e6da5d0 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/incremental-17.5.0.dist-info/entry_points.txt @@ -0,0 +1,4 @@ + + [distutils.setup_keywords] + use_incremental = incremental:_get_version + \ No newline at end of file diff --git a/WebCrawler/venv/Lib/site-packages/incremental-17.5.0.dist-info/metadata.json b/WebCrawler/venv/Lib/site-packages/incremental-17.5.0.dist-info/metadata.json new file mode 100644 index 0000000..649dbdb --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/incremental-17.5.0.dist-info/metadata.json @@ -0,0 +1 @@ +{"classifiers": ["Intended Audience :: Developers", "License :: OSI Approved :: MIT License", "Programming Language :: Python :: 2", "Programming Language :: Python :: 2.6", "Programming Language :: Python :: 2.7", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.3", "Programming Language :: Python :: 3.4", "Programming Language :: Python :: 3.5", "Programming Language :: Python :: 3.6"], "extensions": {"python.details": {"contacts": [{"email": "hawkowl@twistedmatrix.com", "name": "Amber Brown", "role": "author"}], "document_names": {"description": "DESCRIPTION.rst"}, "project_urls": {"Home": "https://github.com/twisted/incremental"}}, "python.exports": {"distutils.setup_keywords": {"use_incremental": "incremental:_get_version"}}}, "extras": ["scripts"], "generator": "bdist_wheel (0.29.0)", "license": "MIT", "metadata_version": "2.0", "name": "incremental", "run_requires": [{"extra": "scripts", "requires": ["click (>=6.0)", "twisted (>=16.4.0)"]}], "summary": "UNKNOWN", "version": "17.5.0"} \ No newline at end of file diff --git a/WebCrawler/venv/Lib/site-packages/incremental-17.5.0.dist-info/top_level.txt b/WebCrawler/venv/Lib/site-packages/incremental-17.5.0.dist-info/top_level.txt new file mode 100644 index 0000000..cb40239 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/incremental-17.5.0.dist-info/top_level.txt @@ -0,0 +1 @@ +incremental diff --git a/WebCrawler/venv/Lib/site-packages/incremental/__init__.py b/WebCrawler/venv/Lib/site-packages/incremental/__init__.py new file mode 100644 index 0000000..8d6aaba --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/incremental/__init__.py @@ -0,0 +1,353 @@ +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Versions for Python packages. + +See L{Version}. +""" + +from __future__ import division, absolute_import + +import sys +import warnings + +# +# Compat functions +# + +if sys.version_info < (3, 0): + _PY3 = False +else: + _PY3 = True + +try: + _cmp = cmp +except NameError: + def _cmp(a, b): + """ + Compare two objects. + + Returns a negative number if C{a < b}, zero if they are equal, and a + positive number if C{a > b}. + """ + if a < b: + return -1 + elif a == b: + return 0 + else: + return 1 + + +def _comparable(klass): + """ + Class decorator that ensures support for the special C{__cmp__} method. + + On Python 2 this does nothing. + + On Python 3, C{__eq__}, C{__lt__}, etc. methods are added to the class, + relying on C{__cmp__} to implement their comparisons. + """ + # On Python 2, __cmp__ will just work, so no need to add extra methods: + if not _PY3: + return klass + + def __eq__(self, other): + c = self.__cmp__(other) + if c is NotImplemented: + return c + return c == 0 + + def __ne__(self, other): + c = self.__cmp__(other) + if c is NotImplemented: + return c + return c != 0 + + def __lt__(self, other): + c = self.__cmp__(other) + if c is NotImplemented: + return c + return c < 0 + + def __le__(self, other): + c = self.__cmp__(other) + if c is NotImplemented: + return c + return c <= 0 + + def __gt__(self, other): + c = self.__cmp__(other) + if c is NotImplemented: + return c + return c > 0 + + def __ge__(self, other): + c = self.__cmp__(other) + if c is NotImplemented: + return c + return c >= 0 + + klass.__lt__ = __lt__ + klass.__gt__ = __gt__ + klass.__le__ = __le__ + klass.__ge__ = __ge__ + klass.__eq__ = __eq__ + klass.__ne__ = __ne__ + return klass + +# +# Versioning +# + + +@_comparable +class _inf(object): + """ + An object that is bigger than all other objects. + """ + def __cmp__(self, other): + """ + @param other: Another object. + @type other: any + + @return: 0 if other is inf, 1 otherwise. + @rtype: C{int} + """ + if other is _inf: + return 0 + return 1 + + +_inf = _inf() + + +class IncomparableVersions(TypeError): + """ + Two versions could not be compared. + """ + + +@_comparable +class Version(object): + """ + An encapsulation of a version for a project, with support for outputting + PEP-440 compatible version strings. + + This class supports the standard major.minor.micro[rcN] scheme of + versioning. + """ + def __init__(self, package, major, minor, micro, release_candidate=None, + prerelease=None, dev=None): + """ + @param package: Name of the package that this is a version of. + @type package: C{str} + @param major: The major version number. + @type major: C{int} or C{str} (for the "NEXT" symbol) + @param minor: The minor version number. + @type minor: C{int} + @param micro: The micro version number. + @type micro: C{int} + @param release_candidate: The release candidate number. + @type release_candidate: C{int} + @param prerelease: The prerelease number. (Deprecated) + @type prerelease: C{int} + @param dev: The development release number. + @type dev: C{int} + """ + if release_candidate and prerelease: + raise ValueError("Please only return one of these.") + elif prerelease and not release_candidate: + release_candidate = prerelease + warnings.warn(("Passing prerelease to incremental.Version was " + "deprecated in Incremental 16.9.0. Please pass " + "release_candidate instead."), + DeprecationWarning, stacklevel=2) + + if major == "NEXT": + if minor or micro or release_candidate or dev: + raise ValueError(("When using NEXT, all other values except " + "Package must be 0.")) + + self.package = package + self.major = major + self.minor = minor + self.micro = micro + self.release_candidate = release_candidate + self.dev = dev + + @property + def prerelease(self): + warnings.warn(("Accessing incremental.Version.prerelease was " + "deprecated in Incremental 16.9.0. Use " + "Version.release_candidate instead."), + DeprecationWarning, stacklevel=2), + return self.release_candidate + + def public(self): + """ + Return a PEP440-compatible "public" representation of this L{Version}. + + Examples: + + - 14.4.0 + - 1.2.3rc1 + - 14.2.1rc1dev9 + - 16.04.0dev0 + """ + if self.major == "NEXT": + return self.major + + if self.release_candidate is None: + rc = "" + else: + rc = "rc%s" % (self.release_candidate,) + + if self.dev is None: + dev = "" + else: + dev = "dev%s" % (self.dev,) + + return '%r.%d.%d%s%s' % (self.major, + self.minor, + self.micro, + rc, dev) + + base = public + short = public + local = public + + def __repr__(self): + + if self.release_candidate is None: + release_candidate = "" + else: + release_candidate = ", release_candidate=%r" % ( + self.release_candidate,) + + if self.dev is None: + dev = "" + else: + dev = ", dev=%r" % (self.dev,) + + return '%s(%r, %r, %d, %d%s%s)' % ( + self.__class__.__name__, + self.package, + self.major, + self.minor, + self.micro, + release_candidate, + dev) + + def __str__(self): + return '[%s, version %s]' % ( + self.package, + self.short()) + + def __cmp__(self, other): + """ + Compare two versions, considering major versions, minor versions, micro + versions, then release candidates. Package names are case insensitive. + + A version with a release candidate is always less than a version + without a release candidate. If both versions have release candidates, + they will be included in the comparison. + + @param other: Another version. + @type other: L{Version} + + @return: NotImplemented when the other object is not a Version, or one + of -1, 0, or 1. + + @raise IncomparableVersions: when the package names of the versions + differ. + """ + if not isinstance(other, self.__class__): + return NotImplemented + if self.package.lower() != other.package.lower(): + raise IncomparableVersions("%r != %r" + % (self.package, other.package)) + + if self.major == "NEXT": + major = _inf + else: + major = self.major + + if self.release_candidate is None: + release_candidate = _inf + else: + release_candidate = self.release_candidate + + if self.dev is None: + dev = _inf + else: + dev = self.dev + + if other.major == "NEXT": + othermajor = _inf + else: + othermajor = other.major + + if other.release_candidate is None: + otherrc = _inf + else: + otherrc = other.release_candidate + + if other.dev is None: + otherdev = _inf + else: + otherdev = other.dev + + x = _cmp((major, + self.minor, + self.micro, + release_candidate, + dev), + (othermajor, + other.minor, + other.micro, + otherrc, + otherdev)) + return x + + +def getVersionString(version): + """ + Get a friendly string for the given version object. + + @param version: A L{Version} object. + @return: A string containing the package and short version number. + """ + result = '%s %s' % (version.package, version.short()) + return result + + +def _get_version(dist, keyword, value): + """ + Get the version from the package listed in the Distribution. + """ + if not value: + return + + from distutils.command import build_py + + sp_command = build_py.build_py(dist) + sp_command.finalize_options() + + for item in sp_command.find_all_modules(): + if item[1] == "_version": + version_file = {} + + with open(item[2]) as f: + exec(f.read(), version_file) + + dist.metadata.version = version_file["__version__"].public() + return None + + raise Exception("No _version.py found.") + + +from ._version import __version__ # noqa + + +__all__ = ["__version__", "Version", "getVersionString"] diff --git a/WebCrawler/venv/Lib/site-packages/incremental/__pycache__/__init__.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/incremental/__pycache__/__init__.cpython-39.pyc new file mode 100644 index 0000000..0efab93 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/incremental/__pycache__/__init__.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/incremental/__pycache__/_version.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/incremental/__pycache__/_version.cpython-39.pyc new file mode 100644 index 0000000..f527d49 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/incremental/__pycache__/_version.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/incremental/__pycache__/update.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/incremental/__pycache__/update.cpython-39.pyc new file mode 100644 index 0000000..c26f4d6 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/incremental/__pycache__/update.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/incremental/_version.py b/WebCrawler/venv/Lib/site-packages/incremental/_version.py new file mode 100644 index 0000000..b705754 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/incremental/_version.py @@ -0,0 +1,11 @@ +""" +Provides Incremental version information. +""" + +# This file is auto-generated! Do not edit! +# Use `python -m incremental.update Incremental` to change this file. + +from incremental import Version + +__version__ = Version('Incremental', 17, 5, 0) +__all__ = ["__version__"] diff --git a/WebCrawler/venv/Lib/site-packages/incremental/tests/__init__.py b/WebCrawler/venv/Lib/site-packages/incremental/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/WebCrawler/venv/Lib/site-packages/incremental/tests/__pycache__/__init__.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/incremental/tests/__pycache__/__init__.cpython-39.pyc new file mode 100644 index 0000000..fe4bec4 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/incremental/tests/__pycache__/__init__.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/incremental/tests/__pycache__/test_update.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/incremental/tests/__pycache__/test_update.cpython-39.pyc new file mode 100644 index 0000000..0d5bb76 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/incremental/tests/__pycache__/test_update.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/incremental/tests/__pycache__/test_version.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/incremental/tests/__pycache__/test_version.cpython-39.pyc new file mode 100644 index 0000000..8ecf433 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/incremental/tests/__pycache__/test_version.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/incremental/tests/test_update.py b/WebCrawler/venv/Lib/site-packages/incremental/tests/test_update.py new file mode 100644 index 0000000..fd2b31c --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/incremental/tests/test_update.py @@ -0,0 +1,715 @@ +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Tests for L{incremental.update}. +""" + +from __future__ import division, absolute_import + +import sys +import os +import datetime + +from twisted.python.filepath import FilePath +from twisted.python.compat import NativeStringIO +from twisted.trial.unittest import TestCase + +from incremental.update import _run, run + + +class NonCreatedUpdateTests(TestCase): + + def setUp(self): + + self.srcdir = FilePath(self.mktemp()) + self.srcdir.makedirs() + + packagedir = self.srcdir.child('inctestpkg') + packagedir.makedirs() + + packagedir.child('__init__.py').setContent(b""" +from incremental import Version +introduced_in = Version('inctestpkg', 'NEXT', 0, 0).short() +next_released_version = "inctestpkg NEXT" +""") + self.getcwd = lambda: self.srcdir.path + self.packagedir = packagedir + + class Date(object): + year = 2016 + month = 8 + + self.date = Date() + + def test_create(self): + """ + `incremental.update package --create` initialises the version. + """ + self.assertFalse(self.packagedir.child("_version.py").exists()) + + out = [] + _run('inctestpkg', path=None, newversion=None, patch=False, rc=False, + dev=False, create=True, _date=self.date, _getcwd=self.getcwd, + _print=out.append) + + self.assertTrue(self.packagedir.child("_version.py").exists()) + self.assertEqual(self.packagedir.child("_version.py").getContent(), + b'''""" +Provides inctestpkg version information. +""" + +# This file is auto-generated! Do not edit! +# Use `python -m incremental.update inctestpkg` to change this file. + +from incremental import Version + +__version__ = Version('inctestpkg', 16, 8, 0) +__all__ = ["__version__"] +''') + + +class MissingTests(TestCase): + + def setUp(self): + + self.srcdir = FilePath(self.mktemp()) + self.srcdir.makedirs() + + self.srcdir.child('srca').makedirs() + + packagedir = self.srcdir.child('srca').child('inctestpkg') + packagedir.makedirs() + + packagedir.child('__init__.py').setContent(b""" +from incremental import Version +introduced_in = Version('inctestpkg', 'NEXT', 0, 0).short() +next_released_version = "inctestpkg NEXT" +""") + packagedir.child('_version.py').setContent(b""" +from incremental import Version +__version__ = Version('inctestpkg', 1, 2, 3) +__all__ = ["__version__"] +""") + self.getcwd = lambda: self.srcdir.path + self.packagedir = packagedir + + class Date(object): + year = 2016 + month = 8 + + self.date = Date() + + def test_path(self): + """ + `incremental.update package --dev` raises and quits if it can't find + the package. + """ + out = [] + with self.assertRaises(ValueError): + _run(u'inctestpkg', path=None, newversion=None, + patch=False, rc=False, dev=True, create=False, + _date=self.date, _getcwd=self.getcwd, _print=out.append) + + +class CreatedUpdateInSrcTests(TestCase): + + def setUp(self): + + self.srcdir = FilePath(self.mktemp()) + self.srcdir.makedirs() + + self.srcdir.child('src').makedirs() + + packagedir = self.srcdir.child('src').child('inctestpkg') + packagedir.makedirs() + + packagedir.child('__init__.py').setContent(b""" +from incremental import Version +introduced_in = Version('inctestpkg', 'NEXT', 0, 0).short() +next_released_version = "inctestpkg NEXT" +""") + packagedir.child('_version.py').setContent(b""" +from incremental import Version +__version__ = Version('inctestpkg', 1, 2, 3) +__all__ = ["__version__"] +""") + self.getcwd = lambda: self.srcdir.path + self.packagedir = packagedir + + class Date(object): + year = 2016 + month = 8 + + self.date = Date() + + def test_path(self): + """ + `incremental.update package --path= --dev` increments the dev + version of the package on the given path + """ + out = [] + _run(u'inctestpkg', path=None, newversion=None, + patch=False, rc=False, dev=True, create=False, _date=self.date, + _getcwd=self.getcwd, _print=out.append) + + self.assertTrue(self.packagedir.child("_version.py").exists()) + self.assertEqual(self.packagedir.child("_version.py").getContent(), + b'''""" +Provides inctestpkg version information. +""" + +# This file is auto-generated! Do not edit! +# Use `python -m incremental.update inctestpkg` to change this file. + +from incremental import Version + +__version__ = Version('inctestpkg', 1, 2, 3, dev=0) +__all__ = ["__version__"] +''') + + _run(u'inctestpkg', path=None, newversion=None, + patch=False, rc=False, dev=True, create=False, _date=self.date, + _getcwd=self.getcwd, _print=out.append) + + self.assertTrue(self.packagedir.child("_version.py").exists()) + self.assertEqual(self.packagedir.child("_version.py").getContent(), + b'''""" +Provides inctestpkg version information. +""" + +# This file is auto-generated! Do not edit! +# Use `python -m incremental.update inctestpkg` to change this file. + +from incremental import Version + +__version__ = Version('inctestpkg', 1, 2, 3, dev=1) +__all__ = ["__version__"] +''') + + +class CreatedUpdateTests(TestCase): + + maxDiff = None + + def setUp(self): + + self.srcdir = FilePath(self.mktemp()) + self.srcdir.makedirs() + + packagedir = self.srcdir.child('inctestpkg') + packagedir.makedirs() + + packagedir.child('__init__.py').setContent(b""" +from incremental import Version +introduced_in = Version('inctestpkg', 'NEXT', 0, 0).short() +next_released_version = "inctestpkg NEXT" +""") + packagedir.child('_version.py').setContent(b""" +from incremental import Version +__version__ = Version('inctestpkg', 1, 2, 3) +__all__ = ["__version__"] +""") + self.getcwd = lambda: self.srcdir.path + self.packagedir = packagedir + + class Date(object): + year = 2016 + month = 8 + + self.date = Date() + + def test_path(self): + """ + `incremental.update package --path= --dev` increments the dev + version of the package on the given path + """ + out = [] + _run(u'inctestpkg', path=self.packagedir.path, newversion=None, + patch=False, rc=False, dev=True, create=False, _date=self.date, + _print=out.append) + + self.assertTrue(self.packagedir.child("_version.py").exists()) + self.assertEqual(self.packagedir.child("_version.py").getContent(), + b'''""" +Provides inctestpkg version information. +""" + +# This file is auto-generated! Do not edit! +# Use `python -m incremental.update inctestpkg` to change this file. + +from incremental import Version + +__version__ = Version('inctestpkg', 1, 2, 3, dev=0) +__all__ = ["__version__"] +''') + + def test_dev(self): + """ + `incremental.update package --dev` increments the dev version. + """ + out = [] + _run(u'inctestpkg', path=None, newversion=None, patch=False, rc=False, + dev=True, create=False, _date=self.date, _getcwd=self.getcwd, + _print=out.append) + + self.assertTrue(self.packagedir.child("_version.py").exists()) + self.assertEqual(self.packagedir.child("_version.py").getContent(), + b'''""" +Provides inctestpkg version information. +""" + +# This file is auto-generated! Do not edit! +# Use `python -m incremental.update inctestpkg` to change this file. + +from incremental import Version + +__version__ = Version('inctestpkg', 1, 2, 3, dev=0) +__all__ = ["__version__"] +''') + + def test_patch(self): + """ + `incremental.update package --patch` increments the patch version. + """ + out = [] + _run(u'inctestpkg', path=None, newversion=None, patch=True, rc=False, + dev=False, create=False, _date=self.date, _getcwd=self.getcwd, + _print=out.append) + + self.assertEqual(self.packagedir.child("_version.py").getContent(), + b'''""" +Provides inctestpkg version information. +""" + +# This file is auto-generated! Do not edit! +# Use `python -m incremental.update inctestpkg` to change this file. + +from incremental import Version + +__version__ = Version('inctestpkg', 1, 2, 4) +__all__ = ["__version__"] +''') + self.assertEqual(self.packagedir.child("__init__.py").getContent(), + b""" +from incremental import Version +introduced_in = Version('inctestpkg', 1, 2, 4).short() +next_released_version = "inctestpkg 1.2.4" +""") + + def test_patch_with_prerelease_and_dev(self): + """ + `incremental.update package --patch` increments the patch version, and + disregards any old prerelease/dev versions. + """ + self.packagedir.child('_version.py').setContent(b""" +from incremental import Version +__version__ = Version('inctestpkg', 1, 2, 3, release_candidate=1, dev=2) +__all__ = ["__version__"] +""") + + out = [] + _run(u'inctestpkg', path=None, newversion=None, patch=True, rc=False, + dev=False, create=False, _date=self.date, _getcwd=self.getcwd, + _print=out.append) + + self.assertEqual(self.packagedir.child("_version.py").getContent(), + b'''""" +Provides inctestpkg version information. +""" + +# This file is auto-generated! Do not edit! +# Use `python -m incremental.update inctestpkg` to change this file. + +from incremental import Version + +__version__ = Version('inctestpkg', 1, 2, 4) +__all__ = ["__version__"] +''') + + def test_rc_patch(self): + """ + `incremental.update package --patch --rc` increments the patch + version and makes it a release candidate. + """ + out = [] + _run(u'inctestpkg', path=None, newversion=None, patch=True, rc=True, + dev=False, create=False, _date=self.date, _getcwd=self.getcwd, + _print=out.append) + + self.assertEqual(self.packagedir.child("_version.py").getContent(), + b'''""" +Provides inctestpkg version information. +""" + +# This file is auto-generated! Do not edit! +# Use `python -m incremental.update inctestpkg` to change this file. + +from incremental import Version + +__version__ = Version('inctestpkg', 1, 2, 4, release_candidate=1) +__all__ = ["__version__"] +''') + self.assertEqual(self.packagedir.child("__init__.py").getContent(), + b""" +from incremental import Version +introduced_in = Version('inctestpkg', 1, 2, 4, release_candidate=1).short() +next_released_version = "inctestpkg 1.2.4rc1" +""") + + def test_rc_with_existing_rc(self): + """ + `incremental.update package --rc` increments the rc version if the + existing version is an rc, and discards any dev version. + """ + self.packagedir.child('_version.py').setContent(b""" +from incremental import Version +__version__ = Version('inctestpkg', 1, 2, 3, release_candidate=1, dev=2) +__all__ = ["__version__"] +""") + + out = [] + _run(u'inctestpkg', path=None, newversion=None, patch=False, rc=True, + dev=False, create=False, _date=self.date, _getcwd=self.getcwd, + _print=out.append) + + self.assertEqual(self.packagedir.child("_version.py").getContent(), + b'''""" +Provides inctestpkg version information. +""" + +# This file is auto-generated! Do not edit! +# Use `python -m incremental.update inctestpkg` to change this file. + +from incremental import Version + +__version__ = Version('inctestpkg', 1, 2, 3, release_candidate=2) +__all__ = ["__version__"] +''') + self.assertEqual(self.packagedir.child("__init__.py").getContent(), + b""" +from incremental import Version +introduced_in = Version('inctestpkg', 1, 2, 3, release_candidate=2).short() +next_released_version = "inctestpkg 1.2.3rc2" +""") + + def test_rc_with_no_rc(self): + """ + `incremental.update package --rc`, when the package is not a release + candidate, will issue a new major/minor rc, and disregards the micro + and dev. + """ + self.packagedir.child('_version.py').setContent(b""" +from incremental import Version +__version__ = Version('inctestpkg', 1, 2, 3, dev=2) +__all__ = ["__version__"] +""") + + out = [] + _run(u'inctestpkg', path=None, newversion=None, patch=False, rc=True, + dev=False, create=False, _date=self.date, _getcwd=self.getcwd, + _print=out.append) + + self.assertEqual(self.packagedir.child("_version.py").getContent(), + b'''""" +Provides inctestpkg version information. +""" + +# This file is auto-generated! Do not edit! +# Use `python -m incremental.update inctestpkg` to change this file. + +from incremental import Version + +__version__ = Version('inctestpkg', 16, 8, 0, release_candidate=1) +__all__ = ["__version__"] +''') + self.assertEqual(self.packagedir.child("__init__.py").getContent(), + b""" +from incremental import Version +introduced_in = Version('inctestpkg', 16, 8, 0, release_candidate=1).short() +next_released_version = "inctestpkg 16.8.0rc1" +""") + + def test_full_with_rc(self): + """ + `incremental.update package`, when the package is a release + candidate, will issue the major/minor, sans release candidate or dev. + """ + out = [] + _run(u'inctestpkg', path=None, newversion=None, patch=False, rc=True, + dev=False, create=False, _date=self.date, _getcwd=self.getcwd, + _print=out.append) + + self.assertEqual(self.packagedir.child("_version.py").getContent(), + b'''""" +Provides inctestpkg version information. +""" + +# This file is auto-generated! Do not edit! +# Use `python -m incremental.update inctestpkg` to change this file. + +from incremental import Version + +__version__ = Version('inctestpkg', 16, 8, 0, release_candidate=1) +__all__ = ["__version__"] +''') + self.assertEqual(self.packagedir.child("__init__.py").getContent(), + b""" +from incremental import Version +introduced_in = Version('inctestpkg', 16, 8, 0, release_candidate=1).short() +next_released_version = "inctestpkg 16.8.0rc1" +""") + + _run(u'inctestpkg', path=None, newversion=None, patch=False, rc=False, + dev=False, create=False, _date=self.date, _getcwd=self.getcwd, + _print=out.append) + + self.assertEqual(self.packagedir.child("_version.py").getContent(), + b'''""" +Provides inctestpkg version information. +""" + +# This file is auto-generated! Do not edit! +# Use `python -m incremental.update inctestpkg` to change this file. + +from incremental import Version + +__version__ = Version('inctestpkg', 16, 8, 0) +__all__ = ["__version__"] +''') + self.assertEqual(self.packagedir.child("__init__.py").getContent(), + b""" +from incremental import Version +introduced_in = Version('inctestpkg', 16, 8, 0).short() +next_released_version = "inctestpkg 16.8.0" +""") + + def test_full_without_rc(self): + """ + `incremental.update package`, when the package is NOT a release + candidate, will raise an error. + """ + out = [] + with self.assertRaises(ValueError) as e: + _run(u'inctestpkg', path=None, newversion=None, patch=False, + rc=False, dev=False, create=False, _date=self.date, + _getcwd=self.getcwd, _print=out.append) + + self.assertEqual( + e.exception.args[0], + "You need to issue a rc before updating the major/minor") + + def test_no_mix_newversion(self): + """ + The `--newversion` flag can't be mixed with --patch, --rc, or --dev. + """ + out = [] + with self.assertRaises(ValueError) as e: + _run(u'inctestpkg', path=None, newversion="1", patch=True, + rc=False, dev=False, create=False, _date=self.date, + _getcwd=self.getcwd, _print=out.append) + self.assertEqual(e.exception.args[0], "Only give --newversion") + + with self.assertRaises(ValueError) as e: + _run(u'inctestpkg', path=None, newversion="1", patch=False, + rc=True, dev=False, create=False, _date=self.date, + _getcwd=self.getcwd, _print=out.append) + self.assertEqual(e.exception.args[0], "Only give --newversion") + + with self.assertRaises(ValueError) as e: + _run(u'inctestpkg', path=None, newversion="1", patch=False, + rc=False, dev=True, create=False, _date=self.date, + _getcwd=self.getcwd, _print=out.append) + self.assertEqual(e.exception.args[0], "Only give --newversion") + + def test_no_mix_dev(self): + """ + The `--dev` flag can't be mixed with --patch, or --rc. + """ + out = [] + with self.assertRaises(ValueError) as e: + _run(u'inctestpkg', path=None, newversion=None, patch=True, + rc=False, dev=True, create=False, _date=self.date, + _getcwd=self.getcwd, _print=out.append) + self.assertEqual(e.exception.args[0], "Only give --dev") + + with self.assertRaises(ValueError) as e: + _run(u'inctestpkg', path=None, newversion=None, patch=False, + rc=True, dev=True, create=False, _date=self.date, + _getcwd=self.getcwd, _print=out.append) + self.assertEqual(e.exception.args[0], "Only give --dev") + + def test_no_mix_create(self): + """ + The `--create` flag can't be mixed with --patch, --rc, --dev, or + --newversion. + """ + out = [] + with self.assertRaises(ValueError) as e: + _run(u'inctestpkg', path=None, newversion=None, patch=True, + rc=False, dev=False, create=True, _date=self.date, + _getcwd=self.getcwd, _print=out.append) + self.assertEqual(e.exception.args[0], "Only give --create") + + with self.assertRaises(ValueError) as e: + _run(u'inctestpkg', path=None, newversion="1", patch=False, + rc=False, dev=False, create=True, _date=self.date, + _getcwd=self.getcwd, _print=out.append) + self.assertEqual(e.exception.args[0], "Only give --create") + + with self.assertRaises(ValueError) as e: + _run(u'inctestpkg', path=None, newversion=None, patch=False, + rc=True, dev=False, create=True, _date=self.date, + _getcwd=self.getcwd, _print=out.append) + self.assertEqual(e.exception.args[0], "Only give --create") + + with self.assertRaises(ValueError) as e: + _run(u'inctestpkg', path=None, newversion=None, patch=False, + rc=False, dev=True, create=True, _date=self.date, + _getcwd=self.getcwd, _print=out.append) + self.assertEqual(e.exception.args[0], "Only give --create") + + def test_newversion(self): + """ + `incremental.update package --newversion=1.2.3rc1dev3`, will set that + version in the package. + """ + out = [] + _run(u'inctestpkg', path=None, newversion="1.2.3rc1dev3", patch=False, + rc=False, dev=False, create=False, _date=self.date, + _getcwd=self.getcwd, _print=out.append) + + self.assertEqual(self.packagedir.child("_version.py").getContent(), + b'''""" +Provides inctestpkg version information. +""" + +# This file is auto-generated! Do not edit! +# Use `python -m incremental.update inctestpkg` to change this file. + +from incremental import Version + +__version__ = Version('inctestpkg', 1, 2, 3, release_candidate=1, dev=3) +__all__ = ["__version__"] +''') + self.assertEqual(self.packagedir.child("__init__.py").getContent(), + (b""" +from incremental import Version +introduced_in = Version('inctestpkg', 1, 2, 3, """ + b"""release_candidate=1, dev=3).short() +next_released_version = "inctestpkg 1.2.3rc1dev3" +""")) + + def test_newversion_bare(self): + """ + `incremental.update package --newversion=1`, will set that + version in the package. + """ + out = [] + _run(u'inctestpkg', path=None, newversion="1", patch=False, + rc=False, dev=False, create=False, _date=self.date, + _getcwd=self.getcwd, _print=out.append) + + self.assertEqual(self.packagedir.child("_version.py").getContent(), + b'''""" +Provides inctestpkg version information. +""" + +# This file is auto-generated! Do not edit! +# Use `python -m incremental.update inctestpkg` to change this file. + +from incremental import Version + +__version__ = Version('inctestpkg', 1, 0, 0) +__all__ = ["__version__"] +''') + self.assertEqual(self.packagedir.child("__init__.py").getContent(), + b""" +from incremental import Version +introduced_in = Version('inctestpkg', 1, 0, 0).short() +next_released_version = "inctestpkg 1.0.0" +""") + + +class ScriptTests(TestCase): + + def setUp(self): + + self.srcdir = FilePath(self.mktemp()) + self.srcdir.makedirs() + + self.srcdir.child('src').makedirs() + + packagedir = self.srcdir.child('src').child('inctestpkg') + packagedir.makedirs() + + packagedir.child('__init__.py').setContent(b""" +from incremental import Version +introduced_in = Version('inctestpkg', 'NEXT', 0, 0).short() +next_released_version = "inctestpkg NEXT" +""") + packagedir.child('_version.py').setContent(b""" +from incremental import Version +__version__ = Version('inctestpkg', 1, 2, 3) +__all__ = ["__version__"] +""") + self.getcwd = lambda: self.srcdir.path + self.packagedir = packagedir + + class Date(object): + year = 2016 + month = 8 + + class DateModule(object): + def today(self): + return Date() + + self.date = DateModule() + + def test_run(self): + """ + Calling run() with no args will cause it to print help. + """ + stringio = NativeStringIO() + self.patch(sys, 'stdout', stringio) + + with self.assertRaises(SystemExit) as e: + run(["--help"]) + + self.assertEqual(e.exception.args[0], 0) + self.assertIn("Show this message and exit", stringio.getvalue()) + + def test_insufficient_args(self): + """ + Calling run() with no args will cause it to print help. + """ + stringio = NativeStringIO() + self.patch(sys, 'stdout', stringio) + self.patch(os, 'getcwd', self.getcwd) + self.patch(datetime, 'date', self.date) + + with self.assertRaises(SystemExit) as e: + run(["inctestpkg", "--rc"]) + + self.assertEqual(e.exception.args[0], 0) + self.assertIn("Updating codebase", stringio.getvalue()) + + self.assertEqual(self.packagedir.child("_version.py").getContent(), + b'''""" +Provides inctestpkg version information. +""" + +# This file is auto-generated! Do not edit! +# Use `python -m incremental.update inctestpkg` to change this file. + +from incremental import Version + +__version__ = Version('inctestpkg', 16, 8, 0, release_candidate=1) +__all__ = ["__version__"] +''') + self.assertEqual(self.packagedir.child("__init__.py").getContent(), + b""" +from incremental import Version +introduced_in = Version('inctestpkg', 16, 8, 0, release_candidate=1).short() +next_released_version = "inctestpkg 16.8.0rc1" +""") diff --git a/WebCrawler/venv/Lib/site-packages/incremental/tests/test_version.py b/WebCrawler/venv/Lib/site-packages/incremental/tests/test_version.py new file mode 100644 index 0000000..d77396c --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/incremental/tests/test_version.py @@ -0,0 +1,415 @@ +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Tests for L{incremental}. +""" + +from __future__ import division, absolute_import + +import operator + +from incremental import getVersionString, IncomparableVersions +from incremental import Version, _inf + +from twisted.trial.unittest import TestCase + + +class VersionsTests(TestCase): + + def test_localIsShort(self): + """ + The local version is the same as the short version. + """ + va = Version("dummy", 1, 0, 0, release_candidate=1, dev=3) + self.assertEqual(va.local(), va.short()) + + def test_versionComparison(self): + """ + Versions can be compared for equality and order. + """ + va = Version("dummy", 1, 0, 0) + vb = Version("dummy", 0, 1, 0) + self.assertTrue(va > vb) + self.assertTrue(vb < va) + self.assertTrue(va >= vb) + self.assertTrue(vb <= va) + self.assertTrue(va != vb) + self.assertTrue(vb == Version("dummy", 0, 1, 0)) + self.assertTrue(vb == vb) + + def test_versionComparisonCaseInsensitive(self): + """ + Version package names are case insensitive. + """ + va = Version("dummy", 1, 0, 0) + vb = Version("DuMmY", 0, 1, 0) + self.assertTrue(va > vb) + self.assertTrue(vb < va) + self.assertTrue(va >= vb) + self.assertTrue(vb <= va) + self.assertTrue(va != vb) + self.assertTrue(vb == Version("dummy", 0, 1, 0)) + self.assertTrue(vb == vb) + + def test_comparingNEXTReleases(self): + """ + NEXT releases are always larger than numbered releases. + """ + va = Version("whatever", "NEXT", 0, 0) + vb = Version("whatever", 1, 0, 0) + self.assertTrue(va > vb) + self.assertFalse(va < vb) + self.assertNotEquals(vb, va) + + def test_NEXTMustBeAlone(self): + """ + NEXT releases must always have the rest of the numbers set to 0. + """ + with self.assertRaises(ValueError): + Version("whatever", "NEXT", 1, 0, release_candidate=0, dev=0) + + with self.assertRaises(ValueError): + Version("whatever", "NEXT", 0, 1, release_candidate=0, dev=0) + + with self.assertRaises(ValueError): + Version("whatever", "NEXT", 0, 0, release_candidate=1, dev=0) + + with self.assertRaises(ValueError): + Version("whatever", "NEXT", 0, 0, release_candidate=0, dev=1) + + def test_comparingNEXTReleasesEqual(self): + """ + NEXT releases are equal to each other. + """ + va = Version("whatever", "NEXT", 0, 0) + vb = Version("whatever", "NEXT", 0, 0) + self.assertEquals(vb, va) + + def test_comparingPrereleasesWithReleases(self): + """ + Prereleases are always less than versions without prereleases. + """ + va = Version("whatever", 1, 0, 0, prerelease=1) + vb = Version("whatever", 1, 0, 0) + self.assertTrue(va < vb) + self.assertFalse(va > vb) + self.assertNotEquals(vb, va) + + def test_prereleaseDeprecated(self): + """ + Passing 'prerelease' to Version is deprecated. + """ + Version("whatever", 1, 0, 0, prerelease=1) + warnings = self.flushWarnings([self.test_prereleaseDeprecated]) + self.assertEqual(len(warnings), 1) + self.assertEqual( + warnings[0]['message'], + ("Passing prerelease to incremental.Version was deprecated in " + "Incremental 16.9.0. Please pass release_candidate instead.")) + + def test_prereleaseAttributeDeprecated(self): + """ + Accessing 'prerelease' on a Version is deprecated. + """ + va = Version("whatever", 1, 0, 0, release_candidate=1) + va.prerelease + warnings = self.flushWarnings( + [self.test_prereleaseAttributeDeprecated]) + self.assertEqual(len(warnings), 1) + self.assertEqual( + warnings[0]['message'], + ("Accessing incremental.Version.prerelease was deprecated in " + "Incremental 16.9.0. Use Version.release_candidate instead.")) + + def test_comparingReleaseCandidatesWithReleases(self): + """ + Release Candidates are always less than versions without release + candidates. + """ + va = Version("whatever", 1, 0, 0, release_candidate=1) + vb = Version("whatever", 1, 0, 0) + self.assertTrue(va < vb) + self.assertFalse(va > vb) + self.assertNotEquals(vb, va) + + def test_comparingDevReleasesWithReleases(self): + """ + Dev releases are always less than versions without dev releases. + """ + va = Version("whatever", 1, 0, 0, dev=1) + vb = Version("whatever", 1, 0, 0) + self.assertTrue(va < vb) + self.assertFalse(va > vb) + self.assertNotEquals(vb, va) + + def test_rcEqualspre(self): + """ + Release Candidates are equal to prereleases. + """ + va = Version("whatever", 1, 0, 0, release_candidate=1) + vb = Version("whatever", 1, 0, 0, prerelease=1) + self.assertTrue(va == vb) + self.assertFalse(va != vb) + + def test_rcOrpreButNotBoth(self): + """ + Release Candidate and prerelease can't both be given. + """ + with self.assertRaises(ValueError): + Version("whatever", 1, 0, 0, + prerelease=1, release_candidate=1) + + def test_comparingReleaseCandidates(self): + """ + The value specified as the release candidate is used in version + comparisons. + """ + va = Version("whatever", 1, 0, 0, release_candidate=1) + vb = Version("whatever", 1, 0, 0, release_candidate=2) + self.assertTrue(va < vb) + self.assertTrue(vb > va) + self.assertTrue(va <= vb) + self.assertTrue(vb >= va) + self.assertTrue(va != vb) + self.assertTrue(vb == Version("whatever", 1, 0, 0, + release_candidate=2)) + self.assertTrue(va == va) + + def test_comparingDev(self): + """ + The value specified as the dev release is used in version comparisons. + """ + va = Version("whatever", 1, 0, 0, dev=1) + vb = Version("whatever", 1, 0, 0, dev=2) + self.assertTrue(va < vb) + self.assertTrue(vb > va) + self.assertTrue(va <= vb) + self.assertTrue(vb >= va) + self.assertTrue(va != vb) + self.assertTrue(vb == Version("whatever", 1, 0, 0, + dev=2)) + self.assertTrue(va == va) + + def test_comparingDevAndRC(self): + """ + The value specified as the dev release and release candidate is used in + version comparisons. + """ + va = Version("whatever", 1, 0, 0, release_candidate=1, dev=1) + vb = Version("whatever", 1, 0, 0, release_candidate=1, dev=2) + self.assertTrue(va < vb) + self.assertTrue(vb > va) + self.assertTrue(va <= vb) + self.assertTrue(vb >= va) + self.assertTrue(va != vb) + self.assertTrue(vb == Version("whatever", 1, 0, 0, + release_candidate=1, dev=2)) + self.assertTrue(va == va) + + def test_comparingDevAndRCDifferent(self): + """ + The value specified as the dev release and release candidate is used in + version comparisons. + """ + va = Version("whatever", 1, 0, 0, release_candidate=1, dev=1) + vb = Version("whatever", 1, 0, 0, release_candidate=2, dev=1) + self.assertTrue(va < vb) + self.assertTrue(vb > va) + self.assertTrue(va <= vb) + self.assertTrue(vb >= va) + self.assertTrue(va != vb) + self.assertTrue(vb == Version("whatever", 1, 0, 0, + release_candidate=2, dev=1)) + self.assertTrue(va == va) + + def test_infComparison(self): + """ + L{_inf} is equal to L{_inf}. + + This is a regression test. + """ + self.assertEqual(_inf, _inf) + + def test_disallowBuggyComparisons(self): + """ + The package names of the Version objects need to be the same. + """ + self.assertRaises(IncomparableVersions, + operator.eq, + Version("dummy", 1, 0, 0), + Version("dumym", 1, 0, 0)) + + def test_notImplementedComparisons(self): + """ + Comparing a L{Version} to some other object type results in + C{NotImplemented}. + """ + va = Version("dummy", 1, 0, 0) + vb = ("dummy", 1, 0, 0) # a tuple is not a Version object + self.assertEqual(va.__cmp__(vb), NotImplemented) + + def test_repr(self): + """ + Calling C{repr} on a version returns a human-readable string + representation of the version. + """ + self.assertEqual(repr(Version("dummy", 1, 2, 3)), + "Version('dummy', 1, 2, 3)") + + def test_reprWithPrerelease(self): + """ + Calling C{repr} on a version with a prerelease returns a human-readable + string representation of the version including the prerelease as a + release candidate.. + """ + self.assertEqual(repr(Version("dummy", 1, 2, 3, prerelease=4)), + "Version('dummy', 1, 2, 3, release_candidate=4)") + + def test_reprWithReleaseCandidate(self): + """ + Calling C{repr} on a version with a release candidate returns a + human-readable string representation of the version including the rc. + """ + self.assertEqual(repr(Version("dummy", 1, 2, 3, release_candidate=4)), + "Version('dummy', 1, 2, 3, release_candidate=4)") + + def test_devWithReleaseCandidate(self): + """ + Calling C{repr} on a version with a dev release returns a + human-readable string representation of the version including the dev + release. + """ + self.assertEqual(repr(Version("dummy", 1, 2, 3, dev=4)), + "Version('dummy', 1, 2, 3, dev=4)") + + def test_str(self): + """ + Calling C{str} on a version returns a human-readable string + representation of the version. + """ + self.assertEqual(str(Version("dummy", 1, 2, 3)), + "[dummy, version 1.2.3]") + + def test_strWithPrerelease(self): + """ + Calling C{str} on a version with a prerelease includes the prerelease + as a release candidate. + """ + self.assertEqual(str(Version("dummy", 1, 0, 0, prerelease=1)), + "[dummy, version 1.0.0rc1]") + + def test_strWithReleaseCandidate(self): + """ + Calling C{str} on a version with a release candidate includes the + release candidate. + """ + self.assertEqual(str(Version("dummy", 1, 0, 0, release_candidate=1)), + "[dummy, version 1.0.0rc1]") + + def test_strWithDevAndReleaseCandidate(self): + """ + Calling C{str} on a version with a release candidate and dev release + includes the release candidate and the dev release. + """ + self.assertEqual(str(Version("dummy", 1, 0, 0, + release_candidate=1, dev=2)), + "[dummy, version 1.0.0rc1dev2]") + + def test_strWithDev(self): + """ + Calling C{str} on a version with a dev release includes the dev + release. + """ + self.assertEqual(str(Version("dummy", 1, 0, 0, dev=1)), + "[dummy, version 1.0.0dev1]") + + def testShort(self): + self.assertEqual(Version('dummy', 1, 2, 3).short(), '1.2.3') + + def test_getVersionString(self): + """ + L{getVersionString} returns a string with the package name and the + short version number. + """ + self.assertEqual( + 'Twisted 8.0.0', getVersionString(Version('Twisted', 8, 0, 0))) + + def test_getVersionStringWithPrerelease(self): + """ + L{getVersionString} includes the prerelease as a release candidate, if + any. + """ + self.assertEqual( + getVersionString(Version("whatever", 8, 0, 0, prerelease=1)), + "whatever 8.0.0rc1") + + def test_getVersionStringWithReleaseCandidate(self): + """ + L{getVersionString} includes the release candidate, if any. + """ + self.assertEqual( + getVersionString(Version("whatever", 8, 0, 0, + release_candidate=1)), + "whatever 8.0.0rc1") + + def test_getVersionStringWithDev(self): + """ + L{getVersionString} includes the dev release, if any. + """ + self.assertEqual( + getVersionString(Version("whatever", 8, 0, 0, + dev=1)), + "whatever 8.0.0dev1") + + def test_getVersionStringWithDevAndRC(self): + """ + L{getVersionString} includes the dev release and release candidate, if + any. + """ + self.assertEqual( + getVersionString(Version("whatever", 8, 0, 0, + release_candidate=2, dev=1)), + "whatever 8.0.0rc2dev1") + + def test_baseWithNEXT(self): + """ + The C{base} method returns just "NEXT" when NEXT is the major version. + """ + self.assertEqual(Version("foo", "NEXT", 0, 0).base(), "NEXT") + + def test_base(self): + """ + The C{base} method returns a very simple representation of the version. + """ + self.assertEqual(Version("foo", 1, 0, 0).base(), "1.0.0") + + def test_baseWithPrerelease(self): + """ + The base version includes 'rcX' for versions with prereleases. + """ + self.assertEqual(Version("foo", 1, 0, 0, prerelease=8).base(), + "1.0.0rc8") + + def test_baseWithDev(self): + """ + The base version includes 'devX' for versions with dev releases. + """ + self.assertEqual(Version("foo", 1, 0, 0, dev=8).base(), + "1.0.0dev8") + + def test_baseWithReleaseCandidate(self): + """ + The base version includes 'rcX' for versions with prereleases. + """ + self.assertEqual(Version("foo", 1, 0, 0, release_candidate=8).base(), + "1.0.0rc8") + + def test_baseWithDevAndRC(self): + """ + The base version includes 'rcXdevX' for versions with dev releases and + a release candidate. + """ + self.assertEqual(Version("foo", 1, 0, 0, + release_candidate=2, dev=8).base(), + "1.0.0rc2dev8") diff --git a/WebCrawler/venv/Lib/site-packages/incremental/update.py b/WebCrawler/venv/Lib/site-packages/incremental/update.py new file mode 100644 index 0000000..bc10d37 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/incremental/update.py @@ -0,0 +1,209 @@ +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +from __future__ import absolute_import, division, print_function + +import click +import os +import datetime + +from incremental import Version +from twisted.python.filepath import FilePath + +_VERSIONPY_TEMPLATE = '''""" +Provides %s version information. +""" + +# This file is auto-generated! Do not edit! +# Use `python -m incremental.update %s` to change this file. + +from incremental import Version + +__version__ = %s +__all__ = ["__version__"] +''' + +_YEAR_START = 2000 + + +def _findPath(path, package): + + cwd = FilePath(path) + + src_dir = cwd.child("src").child(package.lower()) + current_dir = cwd.child(package.lower()) + + if src_dir.isdir(): + return src_dir + elif current_dir.isdir(): + return current_dir + else: + raise ValueError(("Can't find under `./src` or `./`. Check the " + "package name is right (note that we expect your " + "package name to be lower cased), or pass it using " + "'--path'.")) + + +def _existing_version(path): + version_info = {} + + with path.child("_version.py").open('r') as f: + exec(f.read(), version_info) + + return version_info["__version__"] + + +def _run(package, path, newversion, patch, rc, dev, create, + _date=None, _getcwd=None, _print=print): + + if not _getcwd: + _getcwd = os.getcwd + + if not _date: + _date = datetime.date.today() + + if type(package) != str: + package = package.encode('utf8') + + if not path: + path = _findPath(_getcwd(), package) + else: + path = FilePath(path) + + if newversion and patch or newversion and dev or newversion and rc: + raise ValueError("Only give --newversion") + + if dev and patch or dev and rc: + raise ValueError("Only give --dev") + + if create and dev or create and patch or create and rc or \ + create and newversion: + raise ValueError("Only give --create") + + if newversion: + from pkg_resources import parse_version + existing = _existing_version(path) + st_version = parse_version(newversion)._version + + release = list(st_version.release) + + if len(release) == 1: + release.append(0) + if len(release) == 2: + release.append(0) + + v = Version( + package, *release, + release_candidate=st_version.pre[1] if st_version.pre else None, + dev=st_version.dev[1] if st_version.dev else None) + + elif create: + v = Version(package, _date.year - _YEAR_START, _date.month, 0) + existing = v + + elif rc and not patch: + existing = _existing_version(path) + + if existing.release_candidate: + v = Version(package, existing.major, existing.minor, + existing.micro, existing.release_candidate + 1) + else: + v = Version(package, _date.year - _YEAR_START, _date.month, 0, 1) + + elif patch: + if rc: + rc = 1 + else: + rc = None + + existing = _existing_version(path) + v = Version(package, existing.major, existing.minor, + existing.micro + 1, rc) + + elif dev: + existing = _existing_version(path) + + if existing.dev is None: + _dev = 0 + else: + _dev = existing.dev + 1 + + v = Version(package, existing.major, existing.minor, + existing.micro, existing.release_candidate, dev=_dev) + + else: + existing = _existing_version(path) + + if existing.release_candidate: + v = Version(package, + existing.major, existing.minor, existing.micro) + else: + raise ValueError( + "You need to issue a rc before updating the major/minor") + + NEXT_repr = repr(Version(package, "NEXT", 0, 0)).split("#")[0] + NEXT_repr_bytes = NEXT_repr.encode('utf8') + + version_repr = repr(v).split("#")[0] + version_repr_bytes = version_repr.encode('utf8') + + existing_version_repr = repr(existing).split("#")[0] + existing_version_repr_bytes = existing_version_repr.encode('utf8') + + _print("Updating codebase to %s" % (v.public())) + + for x in path.walk(): + + if not x.isfile(): + continue + + original_content = x.getContent() + content = original_content + + # Replace previous release_candidate calls to the new one + if existing.release_candidate: + content = content.replace(existing_version_repr_bytes, + version_repr_bytes) + content = content.replace( + (package.encode('utf8') + b" " + + existing.public().encode('utf8')), + (package.encode('utf8') + b" " + + v.public().encode('utf8'))) + + # Replace NEXT Version calls with the new one + content = content.replace(NEXT_repr_bytes, + version_repr_bytes) + content = content.replace(NEXT_repr_bytes.replace(b"'", b'"'), + version_repr_bytes) + + # Replace NEXT with + content = content.replace(package.encode('utf8') + b" NEXT", + (package.encode('utf8') + b" " + + v.public().encode('utf8'))) + + if content != original_content: + _print("Updating %s" % (x.path,)) + with x.open('w') as f: + f.write(content) + + _print("Updating %s/_version.py" % (path.path)) + with path.child("_version.py").open('w') as f: + f.write( + (_VERSIONPY_TEMPLATE % ( + package, package, version_repr)).encode('utf8')) + + +@click.command() +@click.argument('package') +@click.option('--path', default=None) +@click.option('--newversion', default=None) +@click.option('--patch', is_flag=True) +@click.option('--rc', is_flag=True) +@click.option('--dev', is_flag=True) +@click.option('--create', is_flag=True) +def run(*args, **kwargs): + return _run(*args, **kwargs) + + +if __name__ == '__main__': # pragma: no cover + run() diff --git a/WebCrawler/venv/Lib/site-packages/isort-5.6.4.dist-info/INSTALLER b/WebCrawler/venv/Lib/site-packages/isort-5.6.4.dist-info/INSTALLER new file mode 100644 index 0000000..a1b589e --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/isort-5.6.4.dist-info/INSTALLER @@ -0,0 +1 @@ +pip diff --git a/WebCrawler/venv/Lib/site-packages/isort-5.6.4.dist-info/LICENSE b/WebCrawler/venv/Lib/site-packages/isort-5.6.4.dist-info/LICENSE new file mode 100644 index 0000000..b5083a5 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/isort-5.6.4.dist-info/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2013 Timothy Edmund Crosley + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/WebCrawler/venv/Lib/site-packages/isort-5.6.4.dist-info/METADATA b/WebCrawler/venv/Lib/site-packages/isort-5.6.4.dist-info/METADATA new file mode 100644 index 0000000..21ed02e --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/isort-5.6.4.dist-info/METADATA @@ -0,0 +1,712 @@ +Metadata-Version: 2.1 +Name: isort +Version: 5.6.4 +Summary: A Python utility / library to sort Python imports. +Home-page: https://pycqa.github.io/isort/ +License: MIT +Keywords: Refactor,Lint,Imports,Sort,Clean +Author: Timothy Crosley +Author-email: timothy.crosley@gmail.com +Requires-Python: >=3.6,<4.0 +Classifier: Development Status :: 6 - Mature +Classifier: Environment :: Console +Classifier: Intended Audience :: Developers +Classifier: License :: OSI Approved :: MIT License +Classifier: Natural Language :: English +Classifier: Programming Language :: Python +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3 :: Only +Classifier: Programming Language :: Python :: 3.6 +Classifier: Programming Language :: Python :: 3.7 +Classifier: Programming Language :: Python :: 3.8 +Classifier: Programming Language :: Python :: 3.9 +Classifier: Programming Language :: Python :: Implementation :: CPython +Classifier: Programming Language :: Python :: Implementation :: PyPy +Classifier: Topic :: Software Development :: Libraries +Classifier: Topic :: Utilities +Provides-Extra: colors +Provides-Extra: pipfile_deprecated_finder +Provides-Extra: requirements_deprecated_finder +Requires-Dist: colorama (>=0.4.3,<0.5.0); extra == "colors" +Requires-Dist: pip-api; extra == "requirements_deprecated_finder" +Requires-Dist: pipreqs; extra == "pipfile_deprecated_finder" or extra == "requirements_deprecated_finder" +Requires-Dist: requirementslib; extra == "pipfile_deprecated_finder" +Project-URL: Changelog, https://github.com/pycqa/isort/blob/master/CHANGELOG.md +Project-URL: Documentation, https://pycqa.github.io/isort/ +Project-URL: Repository, https://github.com/pycqa/isort +Description-Content-Type: text/markdown + +[![isort - isort your imports, so you don't have to.](https://raw.githubusercontent.com/pycqa/isort/develop/art/logo_large.png)](https://pycqa.github.io/isort/) + +------------------------------------------------------------------------ + +[![PyPI version](https://badge.fury.io/py/isort.svg)](https://badge.fury.io/py/isort) +[![Test Status](https://github.com/pycqa/isort/workflows/Test/badge.svg?branch=develop)](https://github.com/pycqa/isort/actions?query=workflow%3ATest) +[![Lint Status](https://github.com/pycqa/isort/workflows/Lint/badge.svg?branch=develop)](https://github.com/pycqa/isort/actions?query=workflow%3ALint) +[![Code coverage Status](https://codecov.io/gh/pycqa/isort/branch/develop/graph/badge.svg)](https://codecov.io/gh/pycqa/isort) +[![License](https://img.shields.io/github/license/mashape/apistatus.svg)](https://pypi.org/project/isort/) +[![Join the chat at https://gitter.im/timothycrosley/isort](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/timothycrosley/isort?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) +[![Downloads](https://pepy.tech/badge/isort)](https://pepy.tech/project/isort) +[![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black) +[![Imports: isort](https://img.shields.io/badge/%20imports-isort-%231674b1?style=flat&labelColor=ef8336)](https://pycqa.github.io/isort/) +[![DeepSource](https://static.deepsource.io/deepsource-badge-light-mini.svg)](https://deepsource.io/gh/pycqa/isort/?ref=repository-badge) +_________________ + +[Read Latest Documentation](https://pycqa.github.io/isort/) - [Browse GitHub Code Repository](https://github.com/pycqa/isort/) +_________________ + +isort your imports, so you don't have to. + +isort is a Python utility / library to sort imports alphabetically, and +automatically separated into sections and by type. It provides a command line +utility, Python library and [plugins for various +editors](https://github.com/pycqa/isort/wiki/isort-Plugins) to +quickly sort all your imports. It requires Python 3.6+ to run but +supports formatting Python 2 code too. + +[Try isort now from your browser!](https://pycqa.github.io/isort/docs/quick_start/0.-try/) + +![Example Usage](https://raw.github.com/pycqa/isort/develop/example.gif) + +Before isort: + +```python +from my_lib import Object + +import os + +from my_lib import Object3 + +from my_lib import Object2 + +import sys + +from third_party import lib15, lib1, lib2, lib3, lib4, lib5, lib6, lib7, lib8, lib9, lib10, lib11, lib12, lib13, lib14 + +import sys + +from __future__ import absolute_import + +from third_party import lib3 + +print("Hey") +print("yo") +``` + +After isort: + +```python +from __future__ import absolute_import + +import os +import sys + +from third_party import (lib1, lib2, lib3, lib4, lib5, lib6, lib7, lib8, + lib9, lib10, lib11, lib12, lib13, lib14, lib15) + +from my_lib import Object, Object2, Object3 + +print("Hey") +print("yo") +``` + +## Installing isort + +Installing isort is as simple as: + +```bash +pip install isort +``` + +Install isort with requirements.txt support: + +```bash +pip install isort[requirements_deprecated_finder] +``` + +Install isort with Pipfile support: + +```bash +pip install isort[pipfile_deprecated_finder] +``` + +Install isort with both formats support: + +```bash +pip install isort[requirements_deprecated_finder,pipfile_deprecated_finder] +``` + +## Using isort + +**From the command line**: + +```bash +isort mypythonfile.py mypythonfile2.py +``` + +or recursively: + +```bash +isort . +``` + +*which is equivalent to:* + +```bash +isort **/*.py +``` + +or to see the proposed changes without applying them: + +```bash +isort mypythonfile.py --diff +``` + +Finally, to atomically run isort against a project, only applying +changes if they don't introduce syntax errors do: + +```bash +isort --atomic . +``` + +(Note: this is disabled by default as it keeps isort from being able to +run against code written using a different version of Python) + +**From within Python**: + +```python +import isort + +isort.file("pythonfile.py") +``` + +or: + +```python +import isort + +sorted_code = isort.code("import b\nimport a\n") +``` + +## Installing isort's for your preferred text editor + +Several plugins have been written that enable to use isort from within a +variety of text-editors. You can find a full list of them [on the isort +wiki](https://github.com/pycqa/isort/wiki/isort-Plugins). +Additionally, I will enthusiastically accept pull requests that include +plugins for other text editors and add documentation for them as I am +notified. + +## Multi line output modes + +You will notice above the \"multi\_line\_output\" setting. This setting +defines how from imports wrap when they extend past the line\_length +limit and has 12 possible settings: + +**0 - Grid** + +```python +from third_party import (lib1, lib2, lib3, + lib4, lib5, ...) +``` + +**1 - Vertical** + +```python +from third_party import (lib1, + lib2, + lib3 + lib4, + lib5, + ...) +``` + +**2 - Hanging Indent** + +```python +from third_party import \ + lib1, lib2, lib3, \ + lib4, lib5, lib6 +``` + +**3 - Vertical Hanging Indent** + +```python +from third_party import ( + lib1, + lib2, + lib3, + lib4, +) +``` + +**4 - Hanging Grid** + +```python +from third_party import ( + lib1, lib2, lib3, lib4, + lib5, ...) +``` + +**5 - Hanging Grid Grouped** + +```python +from third_party import ( + lib1, lib2, lib3, lib4, + lib5, ... +) +``` + +**6 - Hanging Grid Grouped, No Trailing Comma** + +In Mode 5 isort leaves a single extra space to maintain consistency of +output when a comma is added at the end. Mode 6 is the same - except +that no extra space is maintained leading to the possibility of lines +one character longer. You can enforce a trailing comma by using this in +conjunction with `-tc` or `include_trailing_comma: True`. + +```python +from third_party import ( + lib1, lib2, lib3, lib4, + lib5 +) +``` + +**7 - NOQA** + +```python +from third_party import lib1, lib2, lib3, ... # NOQA +``` + +Alternatively, you can set `force_single_line` to `True` (`-sl` on the +command line) and every import will appear on its own line: + +```python +from third_party import lib1 +from third_party import lib2 +from third_party import lib3 +... +``` + +**8 - Vertical Hanging Indent Bracket** + +Same as Mode 3 - _Vertical Hanging Indent_ but the closing parentheses +on the last line is indented. + +```python +from third_party import ( + lib1, + lib2, + lib3, + lib4, + ) +``` + +**9 - Vertical Prefix From Module Import** + +Starts a new line with the same `from MODULE import ` prefix when lines are longer than the line length limit. + +```python +from third_party import lib1, lib2, lib3 +from third_party import lib4, lib5, lib6 +``` + +**10 - Hanging Indent With Parentheses** + +Same as Mode 2 - _Hanging Indent_ but uses parentheses instead of backslash +for wrapping long lines. + +```python +from third_party import ( + lib1, lib2, lib3, + lib4, lib5, lib6) +``` + +**11 - Backslash Grid** + +Same as Mode 0 - _Grid_ but uses backslashes instead of parentheses to group imports. + +```python +from third_party import lib1, lib2, lib3, \ + lib4, lib5 +``` + +## Indentation + +To change the how constant indents appear - simply change the +indent property with the following accepted formats: + +- Number of spaces you would like. For example: 4 would cause standard + 4 space indentation. +- Tab +- A verbatim string with quotes around it. + +For example: + +```python +" " +``` + +is equivalent to 4. + +For the import styles that use parentheses, you can control whether or +not to include a trailing comma after the last import with the +`include_trailing_comma` option (defaults to `False`). + +## Intelligently Balanced Multi-line Imports + +As of isort 3.1.0 support for balanced multi-line imports has been +added. With this enabled isort will dynamically change the import length +to the one that produces the most balanced grid, while staying below the +maximum import length defined. + +Example: + +```python +from __future__ import (absolute_import, division, + print_function, unicode_literals) +``` + +Will be produced instead of: + +```python +from __future__ import (absolute_import, division, print_function, + unicode_literals) +``` + +To enable this set `balanced_wrapping` to `True` in your config or pass +the `-e` option into the command line utility. + +## Custom Sections and Ordering + +You can change the section order with `sections` option from the default +of: + +```ini +FUTURE,STDLIB,THIRDPARTY,FIRSTPARTY,LOCALFOLDER +``` + +to your preference: + +```ini +sections=FUTURE,STDLIB,FIRSTPARTY,THIRDPARTY,LOCALFOLDER +``` + +You also can define your own sections and their order. + +Example: + +```ini +known_django=django +known_pandas=pandas,numpy +sections=FUTURE,STDLIB,DJANGO,THIRDPARTY,PANDAS,FIRSTPARTY,LOCALFOLDER +``` + +would create two new sections with the specified known modules. + +The `no_lines_before` option will prevent the listed sections from being +split from the previous section by an empty line. + +Example: + +```ini +sections=FUTURE,STDLIB,THIRDPARTY,FIRSTPARTY,LOCALFOLDER +no_lines_before=LOCALFOLDER +``` + +would produce a section with both FIRSTPARTY and LOCALFOLDER modules +combined. + +**IMPORTANT NOTE**: It is very important to know when setting `known` sections that the naming +does not directly map for historical reasons. For custom settings, the only difference is +capitalization (`known_custom=custom` VS `sections=CUSTOM,...`) for all others reference the +following mapping: + + - `known_standard_library` : `STANDARD_LIBRARY` + - `extra_standard_library` : `STANDARD_LIBRARY` # Like known standard library but appends instead of replacing + - `known_future_library` : `FUTURE` + - `known_first_party`: `FIRSTPARTY` + - `known_third_party`: `THIRDPARTY` + - `known_local_folder`: `LOCALFOLDER` + +This will likely be changed in isort 6.0.0+ in a backwards compatible way. + +## Auto-comment import sections + +Some projects prefer to have import sections uniquely titled to aid in +identifying the sections quickly when visually scanning. isort can +automate this as well. To do this simply set the +`import_heading_{section_name}` setting for each section you wish to +have auto commented - to the desired comment. + +For Example: + +```ini +import_heading_stdlib=Standard Library +import_heading_firstparty=My Stuff +``` + +Would lead to output looking like the following: + +```python +# Standard Library +import os +import sys + +import django.settings + +# My Stuff +import myproject.test +``` + +## Ordering by import length + +isort also makes it easy to sort your imports by length, simply by +setting the `length_sort` option to `True`. This will result in the +following output style: + +```python +from evn.util import ( + Pool, + Dict, + Options, + Constant, + DecayDict, + UnexpectedCodePath, +) +``` + +It is also possible to opt-in to sorting imports by length for only +specific sections by using `length_sort_` followed by the section name +as a configuration item, e.g.: + + length_sort_stdlib=1 + +## Controlling how isort sections `from` imports + +By default isort places straight (`import y`) imports above from imports (`from x import y`): + +```python +import b +from a import a # This will always appear below because it is a from import. +``` + +However, if you prefer to keep strict alphabetical sorting you can set [force sort within sections](https://pycqa.github.io/isort/docs/configuration/options/#force-sort-within-sections) to true. Resulting in: + + +```python +from a import a # This will now appear at top because a appears in the alphabet before b +import b +``` + +You can even tell isort to always place from imports on top, instead of the default of placing them on bottom, using [from first](https://pycqa.github.io/isort/docs/configuration/options/#from-first). + +```python +from b import b # If from first is set to True, all from imports will be placed before non-from imports. +import a +``` + +## Skip processing of imports (outside of configuration) + +To make isort ignore a single import simply add a comment at the end of +the import line containing the text `isort:skip`: + +```python +import module # isort:skip +``` + +or: + +```python +from xyz import (abc, # isort:skip + yo, + hey) +``` + +To make isort skip an entire file simply add `isort:skip_file` to the +module's doc string: + +```python +""" my_module.py + Best module ever + + isort:skip_file +""" + +import b +import a +``` + +## Adding an import to multiple files + +isort makes it easy to add an import statement across multiple files, +while being assured it's correctly placed. + +To add an import to all files: + +```bash +isort -a "from __future__ import print_function" *.py +``` + +To add an import only to files that already have imports: + +```bash +isort -a "from __future__ import print_function" --append-only *.py +``` + + +## Removing an import from multiple files + +isort also makes it easy to remove an import from multiple files, +without having to be concerned with how it was originally formatted. + +From the command line: + +```bash +isort --rm "os.system" *.py +``` + +## Using isort to verify code + +The `--check-only` option +------------------------- + +isort can also be used to verify that code is correctly formatted +by running it with `-c`. Any files that contain incorrectly sorted +and/or formatted imports will be outputted to `stderr`. + +```bash +isort **/*.py -c -v + +SUCCESS: /home/timothy/Projects/Open_Source/isort/isort_kate_plugin.py Everything Looks Good! +ERROR: /home/timothy/Projects/Open_Source/isort/isort/isort.py Imports are incorrectly sorted. +``` + +One great place this can be used is with a pre-commit git hook, such as +this one by \@acdha: + + + +This can help to ensure a certain level of code quality throughout a +project. + +Git hook +-------- + +isort provides a hook function that can be integrated into your Git +pre-commit script to check Python code before committing. + +To cause the commit to fail if there are isort errors (strict mode), +include the following in `.git/hooks/pre-commit`: + +```python +#!/usr/bin/env python +import sys +from isort.hooks import git_hook + +sys.exit(git_hook(strict=True, modify=True, lazy=True, settings_file="")) +``` + +If you just want to display warnings, but allow the commit to happen +anyway, call `git_hook` without the strict parameter. If you want to +display warnings, but not also fix the code, call `git_hook` without the +modify parameter. +The `lazy` argument is to support users who are "lazy" to add files +individually to the index and tend to use `git commit -a` instead. +Set it to `True` to ensure all tracked files are properly isorted, +leave it out or set it to `False` to check only files added to your +index. + +If you want to use a specific configuration file for the hook, you can pass its +path to settings_file. If no path is specifically requested, `git_hook` will +search for the configuration file starting at the directory containing the first +staged file, as per `git diff-index` ordering, and going upward in the directory +structure until a valid configuration file is found or +[`MAX_CONFIG_SEARCH_DEPTH`](src/config.py:35) directories are checked. +The settings_file parameter is used to support users who keep their configuration +file in a directory that might not be a parent of all the other files. + +## Setuptools integration + +Upon installation, isort enables a `setuptools` command that checks +Python files declared by your project. + +Running `python setup.py isort` on the command line will check the files +listed in your `py_modules` and `packages`. If any warning is found, the +command will exit with an error code: + +```bash +$ python setup.py isort +``` + +Also, to allow users to be able to use the command without having to +install isort themselves, add isort to the setup\_requires of your +`setup()` like so: + +```python +setup( + name="project", + packages=["project"], + + setup_requires=[ + "isort" + ] +) +``` + +## Spread the word + +[![Imports: isort](https://img.shields.io/badge/%20imports-isort-%231674b1?style=flat&labelColor=ef8336)](https://pycqa.github.io/isort/) + +Place this badge at the top of your repository to let others know your project uses isort. + +For README.md: + +```markdown +[![Imports: isort](https://img.shields.io/badge/%20imports-isort-%231674b1?style=flat&labelColor=ef8336)](https://pycqa.github.io/isort/) +``` + +Or README.rst: + +```rst +.. image:: https://img.shields.io/badge/%20imports-isort-%231674b1?style=flat&labelColor=ef8336 + :target: https://pycqa.github.io/isort/ +``` + +## Security contact information + +To report a security vulnerability, please use the [Tidelift security +contact](https://tidelift.com/security). Tidelift will coordinate the +fix and disclosure. + +## Why isort? + +isort simply stands for import sort. It was originally called +"sortImports" however I got tired of typing the extra characters and +came to the realization camelCase is not pythonic. + +I wrote isort because in an organization I used to work in the manager +came in one day and decided all code must have alphabetically sorted +imports. The code base was huge - and he meant for us to do it by hand. +However, being a programmer - I\'m too lazy to spend 8 hours mindlessly +performing a function, but not too lazy to spend 16 hours automating it. +I was given permission to open source sortImports and here we are :) + +------------------------------------------------------------------------ + +[Get professionally supported isort with the Tidelift +Subscription](https://tidelift.com/subscription/pkg/pypi-isort?utm_source=pypi-isort&utm_medium=referral&utm_campaign=readme) + +Professional support for isort is available as part of the [Tidelift +Subscription](https://tidelift.com/subscription/pkg/pypi-isort?utm_source=pypi-isort&utm_medium=referral&utm_campaign=readme). +Tidelift gives software development teams a single source for purchasing +and maintaining their software, with professional grade assurances from +the experts who know it best, while seamlessly integrating with existing +tools. + +------------------------------------------------------------------------ + +Thanks and I hope you find isort useful! + +~Timothy Crosley + diff --git a/WebCrawler/venv/Lib/site-packages/isort-5.6.4.dist-info/RECORD b/WebCrawler/venv/Lib/site-packages/isort-5.6.4.dist-info/RECORD new file mode 100644 index 0000000..af683df --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/isort-5.6.4.dist-info/RECORD @@ -0,0 +1,96 @@ +../../Scripts/isort.exe,sha256=2xWJaFS-kxQDmmjGEmbqJy0e8yeajLaYoVcCh0W0ROE,106356 +isort-5.6.4.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4 +isort-5.6.4.dist-info/LICENSE,sha256=BjKUABw9Uj26y6ud1UrCKZgnVsyvWSylMkCysM3YIGU,1089 +isort-5.6.4.dist-info/METADATA,sha256=bY4FamQg0TqGLkZbxP5sjdJx-e5P11hpyC-iFWVVuUM,19947 +isort-5.6.4.dist-info/RECORD,, +isort-5.6.4.dist-info/WHEEL,sha256=xSvaL1DM8LOHfdyo0cCcwjZu1tC6CnCsRGWUgazvlbM,83 +isort-5.6.4.dist-info/entry_points.txt,sha256=7dJFJ8ssCeyoMpdKE7RDH-A5qBWWqgcJh_-RSqMIB0U,142 +isort/__init__.py,sha256=u8zdFTPFro_l9J7JzdeNSlu6CU6BboY3fRTuloxbl7c,374 +isort/__main__.py,sha256=iK0trzN9CCXpQX-XPZDZ9JVkm2Lc0q0oiAgsa6FkJb4,36 +isort/__pycache__/__init__.cpython-39.pyc,, +isort/__pycache__/__main__.cpython-39.pyc,, +isort/__pycache__/_version.cpython-39.pyc,, +isort/__pycache__/api.cpython-39.pyc,, +isort/__pycache__/comments.cpython-39.pyc,, +isort/__pycache__/core.cpython-39.pyc,, +isort/__pycache__/exceptions.cpython-39.pyc,, +isort/__pycache__/format.cpython-39.pyc,, +isort/__pycache__/hooks.cpython-39.pyc,, +isort/__pycache__/io.cpython-39.pyc,, +isort/__pycache__/literal.cpython-39.pyc,, +isort/__pycache__/logo.cpython-39.pyc,, +isort/__pycache__/main.cpython-39.pyc,, +isort/__pycache__/output.cpython-39.pyc,, +isort/__pycache__/parse.cpython-39.pyc,, +isort/__pycache__/place.cpython-39.pyc,, +isort/__pycache__/profiles.cpython-39.pyc,, +isort/__pycache__/pylama_isort.cpython-39.pyc,, +isort/__pycache__/sections.cpython-39.pyc,, +isort/__pycache__/settings.cpython-39.pyc,, +isort/__pycache__/setuptools_commands.cpython-39.pyc,, +isort/__pycache__/sorting.cpython-39.pyc,, +isort/__pycache__/utils.cpython-39.pyc,, +isort/__pycache__/wrap.cpython-39.pyc,, +isort/__pycache__/wrap_modes.cpython-39.pyc,, +isort/_future/__init__.py,sha256=wn-Aa4CVe0zZfA_YBTkJqb6LA9HR9NgpAp0uatzNRNs,326 +isort/_future/__pycache__/__init__.cpython-39.pyc,, +isort/_future/__pycache__/_dataclasses.cpython-39.pyc,, +isort/_future/_dataclasses.py,sha256=zrstLAFOKIHRWJjXdZuHJG2EPiyTVFSQuBcsPpXCCnk,44112 +isort/_vendored/toml/LICENSE,sha256=LZKUgj32yJNXyL5JJ_znk2HWVh5e51MtWSbmOTmqpTY,1252 +isort/_vendored/toml/__init__.py,sha256=gKOk-Amczi2juJsOs1D6UEToaPSIIgNh95Yo5N5gneE,703 +isort/_vendored/toml/__pycache__/__init__.cpython-39.pyc,, +isort/_vendored/toml/__pycache__/decoder.cpython-39.pyc,, +isort/_vendored/toml/__pycache__/encoder.cpython-39.pyc,, +isort/_vendored/toml/__pycache__/ordered.cpython-39.pyc,, +isort/_vendored/toml/__pycache__/tz.cpython-39.pyc,, +isort/_vendored/toml/decoder.py,sha256=5etBKNvVLFAR0rhLCJ9fnRTlqkebI4ZQeoJi_myFbd4,37713 +isort/_vendored/toml/encoder.py,sha256=gQOXYnAWo27Jc_przA1FqLX5AgwbdgN-qDHQtKRx300,9668 +isort/_vendored/toml/ordered.py,sha256=aW5woa5xOqR4BjIz9t10_lghxyhF54KQ7FqUNVv7WJ0,334 +isort/_vendored/toml/tz.py,sha256=8TAiXrTqU08sE0ruz2TXH_pFY2rlwNKE47MSE4rDo8Y,618 +isort/_version.py,sha256=0ryPPy_pGvnG-2tow_ZyL1kF53oMzA7cQOo9s_W0fRQ,22 +isort/api.py,sha256=bSuFdTNkiaSx4OYGdDO_w2A4PctagCAI1V85txrhI90,16022 +isort/comments.py,sha256=23uMZZbUn8y3glMW6_WftnEhECvc-4LW4ysEghpYUUU,962 +isort/core.py,sha256=yetJPY4U8VPGghsFPqG0Wi2Zwnd6pyhW8V2wFTp_Uuw,17848 +isort/deprecated/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +isort/deprecated/__pycache__/__init__.cpython-39.pyc,, +isort/deprecated/__pycache__/finders.cpython-39.pyc,, +isort/deprecated/finders.py,sha256=UOb9z4qPmfzCVlME_OtASEOO8_DcE-mEU-oidkFPnRU,14768 +isort/exceptions.py,sha256=C7xcnrkM64TmxhOUMbc6qtMsXYdTYjZPhFp_upOlNbQ,5877 +isort/format.py,sha256=z_Mf1T2IXh9TyNYY9dw-nWSAfsqWxLvzLwTov7nojuM,5111 +isort/hooks.py,sha256=VatU5IHqBFfTAtl4gaIqr7WocV7r-_6ndtDcrnwq_Y8,3146 +isort/io.py,sha256=sMBvc2VaJ9LijzAq7tVGmlRBRx3goxya_K1E5ayoR74,2057 +isort/literal.py,sha256=4jKIU49AQnH89NZd9Av6CL86glpQB0qxAiVK6pgsYNo,3467 +isort/logo.py,sha256=cL3al79O7O0G2viqRMRfBPp0qtRZmJw2nHSCZw8XWdQ,388 +isort/main.py,sha256=atAWyAZaH5NwztIrmATGCpnKc7H1E0RElV3VL8_RPRQ,35851 +isort/output.py,sha256=XyjtAOnYcFIobMZHdllJY-3X-IxfxPps0aD-ezku22M,23979 +isort/parse.py,sha256=-Oi2K9eJJYuRXPGmAGeUnL3TiOvlf3xj9eWUSOPur34,23147 +isort/place.py,sha256=8gzh3xbhlhQzCiZVv-Z-Bi2XXMYQUJCOLjlJIgXnT90,5165 +isort/profiles.py,sha256=B5iORUoSyiltJ8gDLNodmDaYWir9Nblm1R-CvkYKuWk,1601 +isort/pylama_isort.py,sha256=Ci1K6BPgrxSZC0KO2P4dnz-vQgjWDbnpZe_Rt-0ml1I,1165 +isort/sections.py,sha256=xG5bwU4tOIKUmeBBhZ45EIfjP8HgDOx796bPvD5zWCw,297 +isort/settings.py,sha256=I6iwKTe0EGdETfbzyTrzQy8kGj7u610SCFrfcGIQTp4,28393 +isort/setuptools_commands.py,sha256=zj0C96fXMz-iQv9oPViGPU0xidbx5oYMf5VO1vDKHR0,2263 +isort/sorting.py,sha256=gdcSfo2MFWPGfEPOgVN5dJuLFkBbbZR4nQybNjIQByM,3233 +isort/stdlibs/__init__.py,sha256=MgiO4yPeJZ6ieWz5qSw2LuY7pVmRjZUaCqyUaLH5qJQ,64 +isort/stdlibs/__pycache__/__init__.cpython-39.pyc,, +isort/stdlibs/__pycache__/all.cpython-39.pyc,, +isort/stdlibs/__pycache__/py2.cpython-39.pyc,, +isort/stdlibs/__pycache__/py27.cpython-39.pyc,, +isort/stdlibs/__pycache__/py3.cpython-39.pyc,, +isort/stdlibs/__pycache__/py35.cpython-39.pyc,, +isort/stdlibs/__pycache__/py36.cpython-39.pyc,, +isort/stdlibs/__pycache__/py37.cpython-39.pyc,, +isort/stdlibs/__pycache__/py38.cpython-39.pyc,, +isort/stdlibs/__pycache__/py39.cpython-39.pyc,, +isort/stdlibs/all.py,sha256=n8Es1WK6UlupYyVvf1PDjGbionqix-afC3LkY8nzTcw,57 +isort/stdlibs/py2.py,sha256=dTgWTa7ggz1cwN8fuI9eIs9-5nTmkRxG_uO61CGwfXI,41 +isort/stdlibs/py27.py,sha256=-Id4l2pjAOMXUfwDNnIBR2o8I_mW_Ghmuek2b82Bczk,4492 +isort/stdlibs/py3.py,sha256=899xi1UHQAHxkYJU4lx44qbj41Md85V9Bi05uz5zAts,121 +isort/stdlibs/py35.py,sha256=SVZp9jaCVq4kSjbKcVgF8dJttyFCqcl20ydodsmHrqE,3283 +isort/stdlibs/py36.py,sha256=tCGWDZXWlJJI4_845yOhTpIvnU0-a3TouD_xsMEIZ3s,3298 +isort/stdlibs/py37.py,sha256=nYZmN-s3qMmAHHddegQv6U0j4cnAH0e5SmqTiG6mmhQ,3322 +isort/stdlibs/py38.py,sha256=KE_65iAHg7icOv2xSGScdJWjwBZGuSQYfYcTSIoo_d8,3307 +isort/stdlibs/py39.py,sha256=gHmC2xbsvrqqxybV9G7vrKRv7UmZpgt9NybAhR1LANk,3295 +isort/utils.py,sha256=ohk1VB5WvwXzU8TW3qTWBqQcH1QEA-qUH41Zkw_EVnU,665 +isort/wrap.py,sha256=GiQNWBSOtDEVO2o5OSUpSvEc_7D_SOVdpdLuBVhJDNc,5828 +isort/wrap_modes.py,sha256=BCBV4l89JjVqbfL8cI_i29eSPniUoAb__jKEHqZhFGQ,11258 diff --git a/WebCrawler/venv/Lib/site-packages/isort-5.6.4.dist-info/WHEEL b/WebCrawler/venv/Lib/site-packages/isort-5.6.4.dist-info/WHEEL new file mode 100644 index 0000000..bbb3489 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/isort-5.6.4.dist-info/WHEEL @@ -0,0 +1,4 @@ +Wheel-Version: 1.0 +Generator: poetry 1.0.5 +Root-Is-Purelib: true +Tag: py3-none-any diff --git a/WebCrawler/venv/Lib/site-packages/isort-5.6.4.dist-info/entry_points.txt b/WebCrawler/venv/Lib/site-packages/isort-5.6.4.dist-info/entry_points.txt new file mode 100644 index 0000000..b221602 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/isort-5.6.4.dist-info/entry_points.txt @@ -0,0 +1,9 @@ +[console_scripts] +isort=isort.main:main + +[distutils.commands] +isort=isort.main:ISortCommand + +[pylama.linter] +isort=isort.pylama_isort:Linter + diff --git a/WebCrawler/venv/Lib/site-packages/isort/__init__.py b/WebCrawler/venv/Lib/site-packages/isort/__init__.py new file mode 100644 index 0000000..236255d --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/isort/__init__.py @@ -0,0 +1,9 @@ +"""Defines the public isort interface""" +from . import settings +from ._version import __version__ +from .api import check_code_string as check_code +from .api import check_file, check_stream, place_module, place_module_with_reason +from .api import sort_code_string as code +from .api import sort_file as file +from .api import sort_stream as stream +from .settings import Config diff --git a/WebCrawler/venv/Lib/site-packages/isort/__main__.py b/WebCrawler/venv/Lib/site-packages/isort/__main__.py new file mode 100644 index 0000000..94b1d05 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/isort/__main__.py @@ -0,0 +1,3 @@ +from isort.main import main + +main() diff --git a/WebCrawler/venv/Lib/site-packages/isort/__pycache__/__init__.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/isort/__pycache__/__init__.cpython-39.pyc new file mode 100644 index 0000000..97df41e Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/isort/__pycache__/__init__.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/isort/__pycache__/__main__.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/isort/__pycache__/__main__.cpython-39.pyc new file mode 100644 index 0000000..bac37e2 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/isort/__pycache__/__main__.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/isort/__pycache__/_version.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/isort/__pycache__/_version.cpython-39.pyc new file mode 100644 index 0000000..f243a35 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/isort/__pycache__/_version.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/isort/__pycache__/api.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/isort/__pycache__/api.cpython-39.pyc new file mode 100644 index 0000000..5017605 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/isort/__pycache__/api.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/isort/__pycache__/comments.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/isort/__pycache__/comments.cpython-39.pyc new file mode 100644 index 0000000..13d16e3 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/isort/__pycache__/comments.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/isort/__pycache__/core.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/isort/__pycache__/core.cpython-39.pyc new file mode 100644 index 0000000..f60c5fa Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/isort/__pycache__/core.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/isort/__pycache__/exceptions.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/isort/__pycache__/exceptions.cpython-39.pyc new file mode 100644 index 0000000..2773cc9 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/isort/__pycache__/exceptions.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/isort/__pycache__/format.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/isort/__pycache__/format.cpython-39.pyc new file mode 100644 index 0000000..4134101 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/isort/__pycache__/format.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/isort/__pycache__/hooks.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/isort/__pycache__/hooks.cpython-39.pyc new file mode 100644 index 0000000..f7de513 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/isort/__pycache__/hooks.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/isort/__pycache__/io.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/isort/__pycache__/io.cpython-39.pyc new file mode 100644 index 0000000..8874d75 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/isort/__pycache__/io.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/isort/__pycache__/literal.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/isort/__pycache__/literal.cpython-39.pyc new file mode 100644 index 0000000..10c9538 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/isort/__pycache__/literal.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/isort/__pycache__/logo.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/isort/__pycache__/logo.cpython-39.pyc new file mode 100644 index 0000000..37a5006 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/isort/__pycache__/logo.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/isort/__pycache__/main.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/isort/__pycache__/main.cpython-39.pyc new file mode 100644 index 0000000..66fcb8a Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/isort/__pycache__/main.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/isort/__pycache__/output.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/isort/__pycache__/output.cpython-39.pyc new file mode 100644 index 0000000..a4f12a3 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/isort/__pycache__/output.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/isort/__pycache__/parse.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/isort/__pycache__/parse.cpython-39.pyc new file mode 100644 index 0000000..b034bb8 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/isort/__pycache__/parse.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/isort/__pycache__/place.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/isort/__pycache__/place.cpython-39.pyc new file mode 100644 index 0000000..c65c628 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/isort/__pycache__/place.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/isort/__pycache__/profiles.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/isort/__pycache__/profiles.cpython-39.pyc new file mode 100644 index 0000000..a4580d3 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/isort/__pycache__/profiles.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/isort/__pycache__/pylama_isort.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/isort/__pycache__/pylama_isort.cpython-39.pyc new file mode 100644 index 0000000..679bdb3 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/isort/__pycache__/pylama_isort.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/isort/__pycache__/sections.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/isort/__pycache__/sections.cpython-39.pyc new file mode 100644 index 0000000..d674fbc Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/isort/__pycache__/sections.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/isort/__pycache__/settings.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/isort/__pycache__/settings.cpython-39.pyc new file mode 100644 index 0000000..d9c9b10 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/isort/__pycache__/settings.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/isort/__pycache__/setuptools_commands.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/isort/__pycache__/setuptools_commands.cpython-39.pyc new file mode 100644 index 0000000..bd2613a Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/isort/__pycache__/setuptools_commands.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/isort/__pycache__/sorting.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/isort/__pycache__/sorting.cpython-39.pyc new file mode 100644 index 0000000..279dac7 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/isort/__pycache__/sorting.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/isort/__pycache__/utils.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/isort/__pycache__/utils.cpython-39.pyc new file mode 100644 index 0000000..936575c Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/isort/__pycache__/utils.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/isort/__pycache__/wrap.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/isort/__pycache__/wrap.cpython-39.pyc new file mode 100644 index 0000000..94c10bb Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/isort/__pycache__/wrap.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/isort/__pycache__/wrap_modes.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/isort/__pycache__/wrap_modes.cpython-39.pyc new file mode 100644 index 0000000..d9849b2 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/isort/__pycache__/wrap_modes.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/isort/_future/__init__.py b/WebCrawler/venv/Lib/site-packages/isort/_future/__init__.py new file mode 100644 index 0000000..4d9ef4b --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/isort/_future/__init__.py @@ -0,0 +1,12 @@ +import sys + +if sys.version_info.major <= 3 and sys.version_info.minor <= 6: + from . import _dataclasses as dataclasses # type: ignore + +else: + import dataclasses # type: ignore + +dataclass = dataclasses.dataclass # type: ignore +field = dataclasses.field # type: ignore + +__all__ = ["dataclasses", "dataclass", "field"] diff --git a/WebCrawler/venv/Lib/site-packages/isort/_future/__pycache__/__init__.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/isort/_future/__pycache__/__init__.cpython-39.pyc new file mode 100644 index 0000000..74d57f3 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/isort/_future/__pycache__/__init__.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/isort/_future/__pycache__/_dataclasses.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/isort/_future/__pycache__/_dataclasses.cpython-39.pyc new file mode 100644 index 0000000..69be922 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/isort/_future/__pycache__/_dataclasses.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/isort/_future/_dataclasses.py b/WebCrawler/venv/Lib/site-packages/isort/_future/_dataclasses.py new file mode 100644 index 0000000..87acb66 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/isort/_future/_dataclasses.py @@ -0,0 +1,1209 @@ +# type: ignore +# flake8: noqa +# flake8: noqa +"""Backport of Python3.7 dataclasses Library + +Taken directly from here: https://github.com/ericvsmith/dataclasses +Licensed under the Apache License: https://github.com/ericvsmith/dataclasses/blob/master/LICENSE.txt + +Needed due to isorts strict no non-optional requirements stance. + +TODO: Remove once isort only supports 3.7+ +""" +import copy +import inspect +import keyword +import re +import sys +import types + +__all__ = [ + "dataclass", + "field", + "Field", + "FrozenInstanceError", + "InitVar", + "MISSING", + # Helper functions. + "fields", + "asdict", + "astuple", + "make_dataclass", + "replace", + "is_dataclass", +] + +# Conditions for adding methods. The boxes indicate what action the +# dataclass decorator takes. For all of these tables, when I talk +# about init=, repr=, eq=, order=, unsafe_hash=, or frozen=, I'm +# referring to the arguments to the @dataclass decorator. When +# checking if a dunder method already exists, I mean check for an +# entry in the class's __dict__. I never check to see if an attribute +# is defined in a base class. + +# Key: +# +=========+=========================================+ +# + Value | Meaning | +# +=========+=========================================+ +# | | No action: no method is added. | +# +---------+-----------------------------------------+ +# | add | Generated method is added. | +# +---------+-----------------------------------------+ +# | raise | TypeError is raised. | +# +---------+-----------------------------------------+ +# | None | Attribute is set to None. | +# +=========+=========================================+ + +# __init__ +# +# +--- init= parameter +# | +# v | | | +# | no | yes | <--- class has __init__ in __dict__? +# +=======+=======+=======+ +# | False | | | +# +-------+-------+-------+ +# | True | add | | <- the default +# +=======+=======+=======+ + +# __repr__ +# +# +--- repr= parameter +# | +# v | | | +# | no | yes | <--- class has __repr__ in __dict__? +# +=======+=======+=======+ +# | False | | | +# +-------+-------+-------+ +# | True | add | | <- the default +# +=======+=======+=======+ + + +# __setattr__ +# __delattr__ +# +# +--- frozen= parameter +# | +# v | | | +# | no | yes | <--- class has __setattr__ or __delattr__ in __dict__? +# +=======+=======+=======+ +# | False | | | <- the default +# +-------+-------+-------+ +# | True | add | raise | +# +=======+=======+=======+ +# Raise because not adding these methods would break the "frozen-ness" +# of the class. + +# __eq__ +# +# +--- eq= parameter +# | +# v | | | +# | no | yes | <--- class has __eq__ in __dict__? +# +=======+=======+=======+ +# | False | | | +# +-------+-------+-------+ +# | True | add | | <- the default +# +=======+=======+=======+ + +# __lt__ +# __le__ +# __gt__ +# __ge__ +# +# +--- order= parameter +# | +# v | | | +# | no | yes | <--- class has any comparison method in __dict__? +# +=======+=======+=======+ +# | False | | | <- the default +# +-------+-------+-------+ +# | True | add | raise | +# +=======+=======+=======+ +# Raise because to allow this case would interfere with using +# functools.total_ordering. + +# __hash__ + +# +------------------- unsafe_hash= parameter +# | +----------- eq= parameter +# | | +--- frozen= parameter +# | | | +# v v v | | | +# | no | yes | <--- class has explicitly defined __hash__ +# +=======+=======+=======+========+========+ +# | False | False | False | | | No __eq__, use the base class __hash__ +# +-------+-------+-------+--------+--------+ +# | False | False | True | | | No __eq__, use the base class __hash__ +# +-------+-------+-------+--------+--------+ +# | False | True | False | None | | <-- the default, not hashable +# +-------+-------+-------+--------+--------+ +# | False | True | True | add | | Frozen, so hashable, allows override +# +-------+-------+-------+--------+--------+ +# | True | False | False | add | raise | Has no __eq__, but hashable +# +-------+-------+-------+--------+--------+ +# | True | False | True | add | raise | Has no __eq__, but hashable +# +-------+-------+-------+--------+--------+ +# | True | True | False | add | raise | Not frozen, but hashable +# +-------+-------+-------+--------+--------+ +# | True | True | True | add | raise | Frozen, so hashable +# +=======+=======+=======+========+========+ +# For boxes that are blank, __hash__ is untouched and therefore +# inherited from the base class. If the base is object, then +# id-based hashing is used. +# +# Note that a class may already have __hash__=None if it specified an +# __eq__ method in the class body (not one that was created by +# @dataclass). +# +# See _hash_action (below) for a coded version of this table. + + +# Raised when an attempt is made to modify a frozen class. +class FrozenInstanceError(AttributeError): + pass + + +# A sentinel object for default values to signal that a default +# factory will be used. This is given a nice repr() which will appear +# in the function signature of dataclasses' constructors. +class _HAS_DEFAULT_FACTORY_CLASS: + def __repr__(self): + return "" + + +_HAS_DEFAULT_FACTORY = _HAS_DEFAULT_FACTORY_CLASS() + +# A sentinel object to detect if a parameter is supplied or not. Use +# a class to give it a better repr. +class _MISSING_TYPE: + pass + + +MISSING = _MISSING_TYPE() + +# Since most per-field metadata will be unused, create an empty +# read-only proxy that can be shared among all fields. +_EMPTY_METADATA = types.MappingProxyType({}) + +# Markers for the various kinds of fields and pseudo-fields. +class _FIELD_BASE: + def __init__(self, name): + self.name = name + + def __repr__(self): + return self.name + + +_FIELD = _FIELD_BASE("_FIELD") +_FIELD_CLASSVAR = _FIELD_BASE("_FIELD_CLASSVAR") +_FIELD_INITVAR = _FIELD_BASE("_FIELD_INITVAR") + +# The name of an attribute on the class where we store the Field +# objects. Also used to check if a class is a Data Class. +_FIELDS = "__dataclass_fields__" + +# The name of an attribute on the class that stores the parameters to +# @dataclass. +_PARAMS = "__dataclass_params__" + +# The name of the function, that if it exists, is called at the end of +# __init__. +_POST_INIT_NAME = "__post_init__" + +# String regex that string annotations for ClassVar or InitVar must match. +# Allows "identifier.identifier[" or "identifier[". +# https://bugs.python.org/issue33453 for details. +_MODULE_IDENTIFIER_RE = re.compile(r"^(?:\s*(\w+)\s*\.)?\s*(\w+)") + + +class _InitVarMeta(type): + def __getitem__(self, params): + return self + + +class InitVar(metaclass=_InitVarMeta): + pass + + +# Instances of Field are only ever created from within this module, +# and only from the field() function, although Field instances are +# exposed externally as (conceptually) read-only objects. +# +# name and type are filled in after the fact, not in __init__. +# They're not known at the time this class is instantiated, but it's +# convenient if they're available later. +# +# When cls._FIELDS is filled in with a list of Field objects, the name +# and type fields will have been populated. +class Field: + __slots__ = ( + "name", + "type", + "default", + "default_factory", + "repr", + "hash", + "init", + "compare", + "metadata", + "_field_type", # Private: not to be used by user code. + ) + + def __init__(self, default, default_factory, init, repr, hash, compare, metadata): + self.name = None + self.type = None + self.default = default + self.default_factory = default_factory + self.init = init + self.repr = repr + self.hash = hash + self.compare = compare + self.metadata = ( + _EMPTY_METADATA + if metadata is None or len(metadata) == 0 + else types.MappingProxyType(metadata) + ) + self._field_type = None + + def __repr__(self): + return ( + "Field(" + f"name={self.name!r}," + f"type={self.type!r}," + f"default={self.default!r}," + f"default_factory={self.default_factory!r}," + f"init={self.init!r}," + f"repr={self.repr!r}," + f"hash={self.hash!r}," + f"compare={self.compare!r}," + f"metadata={self.metadata!r}," + f"_field_type={self._field_type}" + ")" + ) + + # This is used to support the PEP 487 __set_name__ protocol in the + # case where we're using a field that contains a descriptor as a + # defaul value. For details on __set_name__, see + # https://www.python.org/dev/peps/pep-0487/#implementation-details. + # + # Note that in _process_class, this Field object is overwritten + # with the default value, so the end result is a descriptor that + # had __set_name__ called on it at the right time. + def __set_name__(self, owner, name): + func = getattr(type(self.default), "__set_name__", None) + if func: + # There is a __set_name__ method on the descriptor, call + # it. + func(self.default, owner, name) + + +class _DataclassParams: + __slots__ = ("init", "repr", "eq", "order", "unsafe_hash", "frozen") + + def __init__(self, init, repr, eq, order, unsafe_hash, frozen): + self.init = init + self.repr = repr + self.eq = eq + self.order = order + self.unsafe_hash = unsafe_hash + self.frozen = frozen + + def __repr__(self): + return ( + "_DataclassParams(" + f"init={self.init!r}," + f"repr={self.repr!r}," + f"eq={self.eq!r}," + f"order={self.order!r}," + f"unsafe_hash={self.unsafe_hash!r}," + f"frozen={self.frozen!r}" + ")" + ) + + +# This function is used instead of exposing Field creation directly, +# so that a type checker can be told (via overloads) that this is a +# function whose type depends on its parameters. +def field( + *, + default=MISSING, + default_factory=MISSING, + init=True, + repr=True, + hash=None, + compare=True, + metadata=None, +): + """Return an object to identify dataclass fields. + default is the default value of the field. default_factory is a + 0-argument function called to initialize a field's value. If init + is True, the field will be a parameter to the class's __init__() + function. If repr is True, the field will be included in the + object's repr(). If hash is True, the field will be included in + the object's hash(). If compare is True, the field will be used + in comparison functions. metadata, if specified, must be a + mapping which is stored but not otherwise examined by dataclass. + It is an error to specify both default and default_factory. + """ + + if default is not MISSING and default_factory is not MISSING: + raise ValueError("cannot specify both default and default_factory") + return Field(default, default_factory, init, repr, hash, compare, metadata) + + +def _tuple_str(obj_name, fields): + # Return a string representing each field of obj_name as a tuple + # member. So, if fields is ['x', 'y'] and obj_name is "self", + # return "(self.x,self.y)". + + # Special case for the 0-tuple. + if not fields: + return "()" + # Note the trailing comma, needed if this turns out to be a 1-tuple. + return f'({",".join(f"{obj_name}.{f.name}" for f in fields)},)' + + +def _create_fn(name, args, body, *, globals=None, locals=None, return_type=MISSING): + # Note that we mutate locals when exec() is called. Caller + # beware! The only callers are internal to this module, so no + # worries about external callers. + if locals is None: + locals = {} + return_annotation = "" + if return_type is not MISSING: + locals["_return_type"] = return_type + return_annotation = "->_return_type" + args = ",".join(args) + body = "\n".join(f" {b}" for b in body) + + # Compute the text of the entire function. + txt = f"def {name}({args}){return_annotation}:\n{body}" + + exec(txt, globals, locals) # nosec + return locals[name] + + +def _field_assign(frozen, name, value, self_name): + # If we're a frozen class, then assign to our fields in __init__ + # via object.__setattr__. Otherwise, just use a simple + # assignment. + # + # self_name is what "self" is called in this function: don't + # hard-code "self", since that might be a field name. + if frozen: + return f"object.__setattr__({self_name},{name!r},{value})" + return f"{self_name}.{name}={value}" + + +def _field_init(f, frozen, globals, self_name): + # Return the text of the line in the body of __init__ that will + # initialize this field. + + default_name = f"_dflt_{f.name}" + if f.default_factory is not MISSING: + if f.init: + # This field has a default factory. If a parameter is + # given, use it. If not, call the factory. + globals[default_name] = f.default_factory + value = f"{default_name}() " f"if {f.name} is _HAS_DEFAULT_FACTORY " f"else {f.name}" + else: + # This is a field that's not in the __init__ params, but + # has a default factory function. It needs to be + # initialized here by calling the factory function, + # because there's no other way to initialize it. + + # For a field initialized with a default=defaultvalue, the + # class dict just has the default value + # (cls.fieldname=defaultvalue). But that won't work for a + # default factory, the factory must be called in __init__ + # and we must assign that to self.fieldname. We can't + # fall back to the class dict's value, both because it's + # not set, and because it might be different per-class + # (which, after all, is why we have a factory function!). + + globals[default_name] = f.default_factory + value = f"{default_name}()" + else: + # No default factory. + if f.init: + if f.default is MISSING: + # There's no default, just do an assignment. + value = f.name + elif f.default is not MISSING: + globals[default_name] = f.default + value = f.name + else: + # This field does not need initialization. Signify that + # to the caller by returning None. + return None + + # Only test this now, so that we can create variables for the + # default. However, return None to signify that we're not going + # to actually do the assignment statement for InitVars. + if f._field_type == _FIELD_INITVAR: + return None + + # Now, actually generate the field assignment. + return _field_assign(frozen, f.name, value, self_name) + + +def _init_param(f): + # Return the __init__ parameter string for this field. For + # example, the equivalent of 'x:int=3' (except instead of 'int', + # reference a variable set to int, and instead of '3', reference a + # variable set to 3). + if f.default is MISSING and f.default_factory is MISSING: + # There's no default, and no default_factory, just output the + # variable name and type. + default = "" + elif f.default is not MISSING: + # There's a default, this will be the name that's used to look + # it up. + default = f"=_dflt_{f.name}" + elif f.default_factory is not MISSING: + # There's a factory function. Set a marker. + default = "=_HAS_DEFAULT_FACTORY" + return f"{f.name}:_type_{f.name}{default}" + + +def _init_fn(fields, frozen, has_post_init, self_name): + # fields contains both real fields and InitVar pseudo-fields. + + # Make sure we don't have fields without defaults following fields + # with defaults. This actually would be caught when exec-ing the + # function source code, but catching it here gives a better error + # message, and future-proofs us in case we build up the function + # using ast. + seen_default = False + for f in fields: + # Only consider fields in the __init__ call. + if f.init: + if not (f.default is MISSING and f.default_factory is MISSING): + seen_default = True + elif seen_default: + raise TypeError(f"non-default argument {f.name!r} " "follows default argument") + + globals = {"MISSING": MISSING, "_HAS_DEFAULT_FACTORY": _HAS_DEFAULT_FACTORY} + + body_lines = [] + for f in fields: + line = _field_init(f, frozen, globals, self_name) + # line is None means that this field doesn't require + # initialization (it's a pseudo-field). Just skip it. + if line: + body_lines.append(line) + + # Does this class have a post-init function? + if has_post_init: + params_str = ",".join(f.name for f in fields if f._field_type is _FIELD_INITVAR) + body_lines.append(f"{self_name}.{_POST_INIT_NAME}({params_str})") + + # If no body lines, use 'pass'. + if not body_lines: + body_lines = ["pass"] + + locals = {f"_type_{f.name}": f.type for f in fields} + return _create_fn( + "__init__", + [self_name] + [_init_param(f) for f in fields if f.init], + body_lines, + locals=locals, + globals=globals, + return_type=None, + ) + + +def _repr_fn(fields): + return _create_fn( + "__repr__", + ("self",), + [ + 'return self.__class__.__qualname__ + f"(' + + ", ".join(f"{f.name}={{self.{f.name}!r}}" for f in fields) + + ')"' + ], + ) + + +def _frozen_get_del_attr(cls, fields): + # XXX: globals is modified on the first call to _create_fn, then + # the modified version is used in the second call. Is this okay? + globals = {"cls": cls, "FrozenInstanceError": FrozenInstanceError} + if fields: + fields_str = "(" + ",".join(repr(f.name) for f in fields) + ",)" + else: + # Special case for the zero-length tuple. + fields_str = "()" + return ( + _create_fn( + "__setattr__", + ("self", "name", "value"), + ( + f"if type(self) is cls or name in {fields_str}:", + ' raise FrozenInstanceError(f"cannot assign to field {name!r}")', + f"super(cls, self).__setattr__(name, value)", + ), + globals=globals, + ), + _create_fn( + "__delattr__", + ("self", "name"), + ( + f"if type(self) is cls or name in {fields_str}:", + ' raise FrozenInstanceError(f"cannot delete field {name!r}")', + f"super(cls, self).__delattr__(name)", + ), + globals=globals, + ), + ) + + +def _cmp_fn(name, op, self_tuple, other_tuple): + # Create a comparison function. If the fields in the object are + # named 'x' and 'y', then self_tuple is the string + # '(self.x,self.y)' and other_tuple is the string + # '(other.x,other.y)'. + + return _create_fn( + name, + ("self", "other"), + [ + "if other.__class__ is self.__class__:", + f" return {self_tuple}{op}{other_tuple}", + "return NotImplemented", + ], + ) + + +def _hash_fn(fields): + self_tuple = _tuple_str("self", fields) + return _create_fn("__hash__", ("self",), [f"return hash({self_tuple})"]) + + +def _is_classvar(a_type, typing): + # This test uses a typing internal class, but it's the best way to + # test if this is a ClassVar. + return type(a_type) is typing._ClassVar + + +def _is_initvar(a_type, dataclasses): + # The module we're checking against is the module we're + # currently in (dataclasses.py). + return a_type is dataclasses.InitVar + + +def _is_type(annotation, cls, a_module, a_type, is_type_predicate): + # Given a type annotation string, does it refer to a_type in + # a_module? For example, when checking that annotation denotes a + # ClassVar, then a_module is typing, and a_type is + # typing.ClassVar. + + # It's possible to look up a_module given a_type, but it involves + # looking in sys.modules (again!), and seems like a waste since + # the caller already knows a_module. + + # - annotation is a string type annotation + # - cls is the class that this annotation was found in + # - a_module is the module we want to match + # - a_type is the type in that module we want to match + # - is_type_predicate is a function called with (obj, a_module) + # that determines if obj is of the desired type. + + # Since this test does not do a local namespace lookup (and + # instead only a module (global) lookup), there are some things it + # gets wrong. + + # With string annotations, cv0 will be detected as a ClassVar: + # CV = ClassVar + # @dataclass + # class C0: + # cv0: CV + + # But in this example cv1 will not be detected as a ClassVar: + # @dataclass + # class C1: + # CV = ClassVar + # cv1: CV + + # In C1, the code in this function (_is_type) will look up "CV" in + # the module and not find it, so it will not consider cv1 as a + # ClassVar. This is a fairly obscure corner case, and the best + # way to fix it would be to eval() the string "CV" with the + # correct global and local namespaces. However that would involve + # a eval() penalty for every single field of every dataclass + # that's defined. It was judged not worth it. + + match = _MODULE_IDENTIFIER_RE.match(annotation) + if match: + ns = None + module_name = match.group(1) + if not module_name: + # No module name, assume the class's module did + # "from dataclasses import InitVar". + ns = sys.modules.get(cls.__module__).__dict__ + else: + # Look up module_name in the class's module. + module = sys.modules.get(cls.__module__) + if module and module.__dict__.get(module_name) is a_module: + ns = sys.modules.get(a_type.__module__).__dict__ + if ns and is_type_predicate(ns.get(match.group(2)), a_module): + return True + return False + + +def _get_field(cls, a_name, a_type): + # Return a Field object for this field name and type. ClassVars + # and InitVars are also returned, but marked as such (see + # f._field_type). + + # If the default value isn't derived from Field, then it's only a + # normal default value. Convert it to a Field(). + default = getattr(cls, a_name, MISSING) + if isinstance(default, Field): + f = default + else: + if isinstance(default, types.MemberDescriptorType): + # This is a field in __slots__, so it has no default value. + default = MISSING + f = field(default=default) + + # Only at this point do we know the name and the type. Set them. + f.name = a_name + f.type = a_type + + # Assume it's a normal field until proven otherwise. We're next + # going to decide if it's a ClassVar or InitVar, everything else + # is just a normal field. + f._field_type = _FIELD + + # In addition to checking for actual types here, also check for + # string annotations. get_type_hints() won't always work for us + # (see https://github.com/python/typing/issues/508 for example), + # plus it's expensive and would require an eval for every stirng + # annotation. So, make a best effort to see if this is a ClassVar + # or InitVar using regex's and checking that the thing referenced + # is actually of the correct type. + + # For the complete discussion, see https://bugs.python.org/issue33453 + + # If typing has not been imported, then it's impossible for any + # annotation to be a ClassVar. So, only look for ClassVar if + # typing has been imported by any module (not necessarily cls's + # module). + typing = sys.modules.get("typing") + if typing: + if _is_classvar(a_type, typing) or ( + isinstance(f.type, str) and _is_type(f.type, cls, typing, typing.ClassVar, _is_classvar) + ): + f._field_type = _FIELD_CLASSVAR + + # If the type is InitVar, or if it's a matching string annotation, + # then it's an InitVar. + if f._field_type is _FIELD: + # The module we're checking against is the module we're + # currently in (dataclasses.py). + dataclasses = sys.modules[__name__] + if _is_initvar(a_type, dataclasses) or ( + isinstance(f.type, str) + and _is_type(f.type, cls, dataclasses, dataclasses.InitVar, _is_initvar) + ): + f._field_type = _FIELD_INITVAR + + # Validations for individual fields. This is delayed until now, + # instead of in the Field() constructor, since only here do we + # know the field name, which allows for better error reporting. + + # Special restrictions for ClassVar and InitVar. + if f._field_type in (_FIELD_CLASSVAR, _FIELD_INITVAR): + if f.default_factory is not MISSING: + raise TypeError(f"field {f.name} cannot have a " "default factory") + # Should I check for other field settings? default_factory + # seems the most serious to check for. Maybe add others. For + # example, how about init=False (or really, + # init=)? It makes no sense for + # ClassVar and InitVar to specify init=. + + # For real fields, disallow mutable defaults for known types. + if f._field_type is _FIELD and isinstance(f.default, (list, dict, set)): + raise ValueError( + f"mutable default {type(f.default)} for field " + f"{f.name} is not allowed: use default_factory" + ) + + return f + + +def _set_new_attribute(cls, name, value): + # Never overwrites an existing attribute. Returns True if the + # attribute already exists. + if name in cls.__dict__: + return True + setattr(cls, name, value) + return False + + +# Decide if/how we're going to create a hash function. Key is +# (unsafe_hash, eq, frozen, does-hash-exist). Value is the action to +# take. The common case is to do nothing, so instead of providing a +# function that is a no-op, use None to signify that. + + +def _hash_set_none(cls, fields): + return None + + +def _hash_add(cls, fields): + flds = [f for f in fields if (f.compare if f.hash is None else f.hash)] + return _hash_fn(flds) + + +def _hash_exception(cls, fields): + # Raise an exception. + raise TypeError(f"Cannot overwrite attribute __hash__ " f"in class {cls.__name__}") + + +# +# +-------------------------------------- unsafe_hash? +# | +------------------------------- eq? +# | | +------------------------ frozen? +# | | | +---------------- has-explicit-hash? +# | | | | +# | | | | +------- action +# | | | | | +# v v v v v +_hash_action = { + (False, False, False, False): None, + (False, False, False, True): None, + (False, False, True, False): None, + (False, False, True, True): None, + (False, True, False, False): _hash_set_none, + (False, True, False, True): None, + (False, True, True, False): _hash_add, + (False, True, True, True): None, + (True, False, False, False): _hash_add, + (True, False, False, True): _hash_exception, + (True, False, True, False): _hash_add, + (True, False, True, True): _hash_exception, + (True, True, False, False): _hash_add, + (True, True, False, True): _hash_exception, + (True, True, True, False): _hash_add, + (True, True, True, True): _hash_exception, +} +# See https://bugs.python.org/issue32929#msg312829 for an if-statement +# version of this table. + + +def _process_class(cls, init, repr, eq, order, unsafe_hash, frozen): + # Now that dicts retain insertion order, there's no reason to use + # an ordered dict. I am leveraging that ordering here, because + # derived class fields overwrite base class fields, but the order + # is defined by the base class, which is found first. + fields = {} + + setattr(cls, _PARAMS, _DataclassParams(init, repr, eq, order, unsafe_hash, frozen)) + + # Find our base classes in reverse MRO order, and exclude + # ourselves. In reversed order so that more derived classes + # override earlier field definitions in base classes. As long as + # we're iterating over them, see if any are frozen. + any_frozen_base = False + has_dataclass_bases = False + for b in cls.__mro__[-1:0:-1]: + # Only process classes that have been processed by our + # decorator. That is, they have a _FIELDS attribute. + base_fields = getattr(b, _FIELDS, None) + if base_fields: + has_dataclass_bases = True + for f in base_fields.values(): + fields[f.name] = f + if getattr(b, _PARAMS).frozen: + any_frozen_base = True + + # Annotations that are defined in this class (not in base + # classes). If __annotations__ isn't present, then this class + # adds no new annotations. We use this to compute fields that are + # added by this class. + # + # Fields are found from cls_annotations, which is guaranteed to be + # ordered. Default values are from class attributes, if a field + # has a default. If the default value is a Field(), then it + # contains additional info beyond (and possibly including) the + # actual default value. Pseudo-fields ClassVars and InitVars are + # included, despite the fact that they're not real fields. That's + # dealt with later. + cls_annotations = cls.__dict__.get("__annotations__", {}) + + # Now find fields in our class. While doing so, validate some + # things, and set the default values (as class attributes) where + # we can. + cls_fields = [_get_field(cls, name, type) for name, type in cls_annotations.items()] + for f in cls_fields: + fields[f.name] = f + + # If the class attribute (which is the default value for this + # field) exists and is of type 'Field', replace it with the + # real default. This is so that normal class introspection + # sees a real default value, not a Field. + if isinstance(getattr(cls, f.name, None), Field): + if f.default is MISSING: + # If there's no default, delete the class attribute. + # This happens if we specify field(repr=False), for + # example (that is, we specified a field object, but + # no default value). Also if we're using a default + # factory. The class attribute should not be set at + # all in the post-processed class. + delattr(cls, f.name) + else: + setattr(cls, f.name, f.default) + + # Do we have any Field members that don't also have annotations? + for name, value in cls.__dict__.items(): + if isinstance(value, Field) and not name in cls_annotations: + raise TypeError(f"{name!r} is a field but has no type annotation") + + # Check rules that apply if we are derived from any dataclasses. + if has_dataclass_bases: + # Raise an exception if any of our bases are frozen, but we're not. + if any_frozen_base and not frozen: + raise TypeError("cannot inherit non-frozen dataclass from a " "frozen one") + + # Raise an exception if we're frozen, but none of our bases are. + if not any_frozen_base and frozen: + raise TypeError("cannot inherit frozen dataclass from a " "non-frozen one") + + # Remember all of the fields on our class (including bases). This + # also marks this class as being a dataclass. + setattr(cls, _FIELDS, fields) + + # Was this class defined with an explicit __hash__? Note that if + # __eq__ is defined in this class, then python will automatically + # set __hash__ to None. This is a heuristic, as it's possible + # that such a __hash__ == None was not auto-generated, but it + # close enough. + class_hash = cls.__dict__.get("__hash__", MISSING) + has_explicit_hash = not ( + class_hash is MISSING or (class_hash is None and "__eq__" in cls.__dict__) + ) + + # If we're generating ordering methods, we must be generating the + # eq methods. + if order and not eq: + raise ValueError("eq must be true if order is true") + + if init: + # Does this class have a post-init function? + has_post_init = hasattr(cls, _POST_INIT_NAME) + + # Include InitVars and regular fields (so, not ClassVars). + flds = [f for f in fields.values() if f._field_type in (_FIELD, _FIELD_INITVAR)] + _set_new_attribute( + cls, + "__init__", + _init_fn( + flds, + frozen, + has_post_init, + # The name to use for the "self" + # param in __init__. Use "self" + # if possible. + "__dataclass_self__" if "self" in fields else "self", + ), + ) + + # Get the fields as a list, and include only real fields. This is + # used in all of the following methods. + field_list = [f for f in fields.values() if f._field_type is _FIELD] + + if repr: + flds = [f for f in field_list if f.repr] + _set_new_attribute(cls, "__repr__", _repr_fn(flds)) + + if eq: + # Create _eq__ method. There's no need for a __ne__ method, + # since python will call __eq__ and negate it. + flds = [f for f in field_list if f.compare] + self_tuple = _tuple_str("self", flds) + other_tuple = _tuple_str("other", flds) + _set_new_attribute(cls, "__eq__", _cmp_fn("__eq__", "==", self_tuple, other_tuple)) + + if order: + # Create and set the ordering methods. + flds = [f for f in field_list if f.compare] + self_tuple = _tuple_str("self", flds) + other_tuple = _tuple_str("other", flds) + for name, op in [("__lt__", "<"), ("__le__", "<="), ("__gt__", ">"), ("__ge__", ">=")]: + if _set_new_attribute(cls, name, _cmp_fn(name, op, self_tuple, other_tuple)): + raise TypeError( + f"Cannot overwrite attribute {name} " + f"in class {cls.__name__}. Consider using " + "functools.total_ordering" + ) + + if frozen: + for fn in _frozen_get_del_attr(cls, field_list): + if _set_new_attribute(cls, fn.__name__, fn): + raise TypeError( + f"Cannot overwrite attribute {fn.__name__} " f"in class {cls.__name__}" + ) + + # Decide if/how we're going to create a hash function. + hash_action = _hash_action[bool(unsafe_hash), bool(eq), bool(frozen), has_explicit_hash] + if hash_action: + # No need to call _set_new_attribute here, since by the time + # we're here the overwriting is unconditional. + cls.__hash__ = hash_action(cls, field_list) + + if not getattr(cls, "__doc__"): + # Create a class doc-string. + cls.__doc__ = cls.__name__ + str(inspect.signature(cls)).replace(" -> None", "") + + return cls + + +# _cls should never be specified by keyword, so start it with an +# underscore. The presence of _cls is used to detect if this +# decorator is being called with parameters or not. +def dataclass( + _cls=None, *, init=True, repr=True, eq=True, order=False, unsafe_hash=False, frozen=False +): + """Returns the same class as was passed in, with dunder methods + added based on the fields defined in the class. + Examines PEP 526 __annotations__ to determine fields. + If init is true, an __init__() method is added to the class. If + repr is true, a __repr__() method is added. If order is true, rich + comparison dunder methods are added. If unsafe_hash is true, a + __hash__() method function is added. If frozen is true, fields may + not be assigned to after instance creation. + """ + + def wrap(cls): + return _process_class(cls, init, repr, eq, order, unsafe_hash, frozen) + + # See if we're being called as @dataclass or @dataclass(). + if _cls is None: + # We're called with parens. + return wrap + + # We're called as @dataclass without parens. + return wrap(_cls) + + +def fields(class_or_instance): + """Return a tuple describing the fields of this dataclass. + Accepts a dataclass or an instance of one. Tuple elements are of + type Field. + """ + + # Might it be worth caching this, per class? + try: + fields = getattr(class_or_instance, _FIELDS) + except AttributeError: + raise TypeError("must be called with a dataclass type or instance") + + # Exclude pseudo-fields. Note that fields is sorted by insertion + # order, so the order of the tuple is as the fields were defined. + return tuple(f for f in fields.values() if f._field_type is _FIELD) + + +def _is_dataclass_instance(obj): + """Returns True if obj is an instance of a dataclass.""" + return not isinstance(obj, type) and hasattr(obj, _FIELDS) + + +def is_dataclass(obj): + """Returns True if obj is a dataclass or an instance of a + dataclass.""" + return hasattr(obj, _FIELDS) + + +def asdict(obj, *, dict_factory=dict): + """Return the fields of a dataclass instance as a new dictionary mapping + field names to field values. + Example usage: + @dataclass + class C: + x: int + y: int + c = C(1, 2) + assert asdict(c) == {'x': 1, 'y': 2} + If given, 'dict_factory' will be used instead of built-in dict. + The function applies recursively to field values that are + dataclass instances. This will also look into built-in containers: + tuples, lists, and dicts. + """ + if not _is_dataclass_instance(obj): + raise TypeError("asdict() should be called on dataclass instances") + return _asdict_inner(obj, dict_factory) + + +def _asdict_inner(obj, dict_factory): + if _is_dataclass_instance(obj): + result = [] + for f in fields(obj): + value = _asdict_inner(getattr(obj, f.name), dict_factory) + result.append((f.name, value)) + return dict_factory(result) + elif isinstance(obj, (list, tuple)): + return type(obj)(_asdict_inner(v, dict_factory) for v in obj) + elif isinstance(obj, dict): + return type(obj)( + (_asdict_inner(k, dict_factory), _asdict_inner(v, dict_factory)) for k, v in obj.items() + ) + else: + return copy.deepcopy(obj) + + +def astuple(obj, *, tuple_factory=tuple): + """Return the fields of a dataclass instance as a new tuple of field values. + Example usage:: + @dataclass + class C: + x: int + y: int + c = C(1, 2) + assert astuple(c) == (1, 2) + If given, 'tuple_factory' will be used instead of built-in tuple. + The function applies recursively to field values that are + dataclass instances. This will also look into built-in containers: + tuples, lists, and dicts. + """ + + if not _is_dataclass_instance(obj): + raise TypeError("astuple() should be called on dataclass instances") + return _astuple_inner(obj, tuple_factory) + + +def _astuple_inner(obj, tuple_factory): + if _is_dataclass_instance(obj): + result = [] + for f in fields(obj): + value = _astuple_inner(getattr(obj, f.name), tuple_factory) + result.append(value) + return tuple_factory(result) + elif isinstance(obj, (list, tuple)): + return type(obj)(_astuple_inner(v, tuple_factory) for v in obj) + elif isinstance(obj, dict): + return type(obj)( + (_astuple_inner(k, tuple_factory), _astuple_inner(v, tuple_factory)) + for k, v in obj.items() + ) + else: + return copy.deepcopy(obj) + + +def make_dataclass( + cls_name, + fields, + *, + bases=(), + namespace=None, + init=True, + repr=True, + eq=True, + order=False, + unsafe_hash=False, + frozen=False, +): + """Return a new dynamically created dataclass. + The dataclass name will be 'cls_name'. 'fields' is an iterable + of either (name), (name, type) or (name, type, Field) objects. If type is + omitted, use the string 'typing.Any'. Field objects are created by + the equivalent of calling 'field(name, type [, Field-info])'. + C = make_dataclass('C', ['x', ('y', int), ('z', int, field(init=False))], bases=(Base,)) + is equivalent to: + @dataclass + class C(Base): + x: 'typing.Any' + y: int + z: int = field(init=False) + For the bases and namespace parameters, see the builtin type() function. + The parameters init, repr, eq, order, unsafe_hash, and frozen are passed to + dataclass(). + """ + + if namespace is None: + namespace = {} + else: + # Copy namespace since we're going to mutate it. + namespace = namespace.copy() + + # While we're looking through the field names, validate that they + # are identifiers, are not keywords, and not duplicates. + seen = set() + anns = {} + for item in fields: + if isinstance(item, str): + name = item + tp = "typing.Any" + elif len(item) == 2: + ( + name, + tp, + ) = item + elif len(item) == 3: + name, tp, spec = item + namespace[name] = spec + else: + raise TypeError(f"Invalid field: {item!r}") + + if not isinstance(name, str) or not name.isidentifier(): + raise TypeError(f"Field names must be valid identifers: {name!r}") + if keyword.iskeyword(name): + raise TypeError(f"Field names must not be keywords: {name!r}") + if name in seen: + raise TypeError(f"Field name duplicated: {name!r}") + + seen.add(name) + anns[name] = tp + + namespace["__annotations__"] = anns + # We use `types.new_class()` instead of simply `type()` to allow dynamic creation + # of generic dataclassses. + cls = types.new_class(cls_name, bases, {}, lambda ns: ns.update(namespace)) + return dataclass( + cls, init=init, repr=repr, eq=eq, order=order, unsafe_hash=unsafe_hash, frozen=frozen + ) + + +def replace(obj, **changes): + """Return a new object replacing specified fields with new values. + This is especially useful for frozen classes. Example usage: + @dataclass(frozen=True) + class C: + x: int + y: int + c = C(1, 2) + c1 = replace(c, x=3) + assert c1.x == 3 and c1.y == 2 + """ + + # We're going to mutate 'changes', but that's okay because it's a + # new dict, even if called with 'replace(obj, **my_changes)'. + + if not _is_dataclass_instance(obj): + raise TypeError("replace() should be called on dataclass instances") + + # It's an error to have init=False fields in 'changes'. + # If a field is not in 'changes', read its value from the provided obj. + + for f in getattr(obj, _FIELDS).values(): + if not f.init: + # Error if this field is specified in changes. + if f.name in changes: + raise ValueError( + f"field {f.name} is declared with " + "init=False, it cannot be specified with " + "replace()" + ) + continue + + if f.name not in changes: + changes[f.name] = getattr(obj, f.name) + + # Create the new object, which calls __init__() and + # __post_init__() (if defined), using all of the init fields we've + # added and/or left in 'changes'. If there are values supplied in + # changes that aren't fields, this will correctly raise a + # TypeError. + return obj.__class__(**changes) diff --git a/WebCrawler/venv/Lib/site-packages/isort/_vendored/toml/LICENSE b/WebCrawler/venv/Lib/site-packages/isort/_vendored/toml/LICENSE new file mode 100644 index 0000000..5010e30 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/isort/_vendored/toml/LICENSE @@ -0,0 +1,27 @@ +The MIT License + +Copyright 2013-2019 William Pearson +Copyright 2015-2016 Julien Enselme +Copyright 2016 Google Inc. +Copyright 2017 Samuel Vasko +Copyright 2017 Nate Prewitt +Copyright 2017 Jack Evans +Copyright 2019 Filippo Broggini + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. \ No newline at end of file diff --git a/WebCrawler/venv/Lib/site-packages/isort/_vendored/toml/__init__.py b/WebCrawler/venv/Lib/site-packages/isort/_vendored/toml/__init__.py new file mode 100644 index 0000000..8cefeff --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/isort/_vendored/toml/__init__.py @@ -0,0 +1,23 @@ +"""Python module which parses and emits TOML. + +Released under the MIT license. +""" +from . import decoder, encoder + +__version__ = "0.10.1" +_spec_ = "0.5.0" + +load = decoder.load +loads = decoder.loads +TomlDecoder = decoder.TomlDecoder +TomlDecodeError = decoder.TomlDecodeError +TomlPreserveCommentDecoder = decoder.TomlPreserveCommentDecoder + +dump = encoder.dump +dumps = encoder.dumps +TomlEncoder = encoder.TomlEncoder +TomlArraySeparatorEncoder = encoder.TomlArraySeparatorEncoder +TomlPreserveInlineDictEncoder = encoder.TomlPreserveInlineDictEncoder +TomlNumpyEncoder = encoder.TomlNumpyEncoder +TomlPreserveCommentEncoder = encoder.TomlPreserveCommentEncoder +TomlPathlibEncoder = encoder.TomlPathlibEncoder diff --git a/WebCrawler/venv/Lib/site-packages/isort/_vendored/toml/__pycache__/__init__.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/isort/_vendored/toml/__pycache__/__init__.cpython-39.pyc new file mode 100644 index 0000000..9cf6cb0 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/isort/_vendored/toml/__pycache__/__init__.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/isort/_vendored/toml/__pycache__/decoder.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/isort/_vendored/toml/__pycache__/decoder.cpython-39.pyc new file mode 100644 index 0000000..223e1ba Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/isort/_vendored/toml/__pycache__/decoder.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/isort/_vendored/toml/__pycache__/encoder.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/isort/_vendored/toml/__pycache__/encoder.cpython-39.pyc new file mode 100644 index 0000000..79d14fa Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/isort/_vendored/toml/__pycache__/encoder.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/isort/_vendored/toml/__pycache__/ordered.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/isort/_vendored/toml/__pycache__/ordered.cpython-39.pyc new file mode 100644 index 0000000..74f1ccd Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/isort/_vendored/toml/__pycache__/ordered.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/isort/_vendored/toml/__pycache__/tz.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/isort/_vendored/toml/__pycache__/tz.cpython-39.pyc new file mode 100644 index 0000000..8dc90d0 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/isort/_vendored/toml/__pycache__/tz.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/isort/_vendored/toml/decoder.py b/WebCrawler/venv/Lib/site-packages/isort/_vendored/toml/decoder.py new file mode 100644 index 0000000..b90b693 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/isort/_vendored/toml/decoder.py @@ -0,0 +1,1053 @@ +import datetime +import io +import re +import sys +from os import linesep + +from .tz import TomlTz + +if sys.version_info < (3,): + _range = xrange # noqa: F821 +else: + unicode = str + _range = range + basestring = str + unichr = chr + + +def _detect_pathlib_path(p): + if (3, 4) <= sys.version_info: + import pathlib + + if isinstance(p, pathlib.PurePath): + return True + return False + + +def _ispath(p): + if isinstance(p, (bytes, basestring)): + return True + return _detect_pathlib_path(p) + + +def _getpath(p): + if (3, 6) <= sys.version_info: + import os + + return os.fspath(p) + if _detect_pathlib_path(p): + return str(p) + return p + + +try: + FNFError = FileNotFoundError +except NameError: + FNFError = IOError + + +TIME_RE = re.compile(r"([0-9]{2}):([0-9]{2}):([0-9]{2})(\.([0-9]{3,6}))?") + + +class TomlDecodeError(ValueError): + """Base toml Exception / Error.""" + + def __init__(self, msg, doc, pos): + lineno = doc.count("\n", 0, pos) + 1 + colno = pos - doc.rfind("\n", 0, pos) + emsg = "{} (line {} column {} char {})".format(msg, lineno, colno, pos) + ValueError.__init__(self, emsg) + self.msg = msg + self.doc = doc + self.pos = pos + self.lineno = lineno + self.colno = colno + + +# Matches a TOML number, which allows underscores for readability +_number_with_underscores = re.compile("([0-9])(_([0-9]))*") + + +class CommentValue(object): + def __init__(self, val, comment, beginline, _dict): + self.val = val + separator = "\n" if beginline else " " + self.comment = separator + comment + self._dict = _dict + + def __getitem__(self, key): + return self.val[key] + + def __setitem__(self, key, value): + self.val[key] = value + + def dump(self, dump_value_func): + retstr = dump_value_func(self.val) + if isinstance(self.val, self._dict): + return self.comment + "\n" + unicode(retstr) + else: + return unicode(retstr) + self.comment + + +def _strictly_valid_num(n): + n = n.strip() + if not n: + return False + if n[0] == "_": + return False + if n[-1] == "_": + return False + if "_." in n or "._" in n: + return False + if len(n) == 1: + return True + if n[0] == "0" and n[1] not in [".", "o", "b", "x"]: + return False + if n[0] == "+" or n[0] == "-": + n = n[1:] + if len(n) > 1 and n[0] == "0" and n[1] != ".": + return False + if "__" in n: + return False + return True + + +def load(f, _dict=dict, decoder=None): + """Parses named file or files as toml and returns a dictionary + + Args: + f: Path to the file to open, array of files to read into single dict + or a file descriptor + _dict: (optional) Specifies the class of the returned toml dictionary + decoder: The decoder to use + + Returns: + Parsed toml file represented as a dictionary + + Raises: + TypeError -- When f is invalid type + TomlDecodeError: Error while decoding toml + IOError / FileNotFoundError -- When an array with no valid (existing) + (Python 2 / Python 3) file paths is passed + """ + + if _ispath(f): + with io.open(_getpath(f), encoding="utf-8") as ffile: + return loads(ffile.read(), _dict, decoder) + elif isinstance(f, list): + from os import path as op + from warnings import warn + + if not [path for path in f if op.exists(path)]: + error_msg = "Load expects a list to contain filenames only." + error_msg += linesep + error_msg += "The list needs to contain the path of at least one " "existing file." + raise FNFError(error_msg) + if decoder is None: + decoder = TomlDecoder(_dict) + d = decoder.get_empty_table() + for l in f: # noqa: E741 + if op.exists(l): + d.update(load(l, _dict, decoder)) + else: + warn("Non-existent filename in list with at least one valid " "filename") + return d + else: + try: + return loads(f.read(), _dict, decoder) + except AttributeError: + raise TypeError("You can only load a file descriptor, filename or " "list") + + +_groupname_re = re.compile(r"^[A-Za-z0-9_-]+$") + + +def loads(s, _dict=dict, decoder=None): + """Parses string as toml + + Args: + s: String to be parsed + _dict: (optional) Specifies the class of the returned toml dictionary + + Returns: + Parsed toml file represented as a dictionary + + Raises: + TypeError: When a non-string is passed + TomlDecodeError: Error while decoding toml + """ + + implicitgroups = [] + if decoder is None: + decoder = TomlDecoder(_dict) + retval = decoder.get_empty_table() + currentlevel = retval + if not isinstance(s, basestring): + raise TypeError("Expecting something like a string") + + if not isinstance(s, unicode): + s = s.decode("utf8") + + original = s + sl = list(s) + openarr = 0 + openstring = False + openstrchar = "" + multilinestr = False + arrayoftables = False + beginline = True + keygroup = False + dottedkey = False + keyname = 0 + key = "" + prev_key = "" + line_no = 1 + + for i, item in enumerate(sl): + if item == "\r" and sl[i + 1] == "\n": + sl[i] = " " + continue + if keyname: + key += item + if item == "\n": + raise TomlDecodeError( + "Key name found without value." " Reached end of line.", original, i + ) + if openstring: + if item == openstrchar: + oddbackslash = False + k = 1 + while i >= k and sl[i - k] == "\\": + oddbackslash = not oddbackslash + k += 1 + if not oddbackslash: + keyname = 2 + openstring = False + openstrchar = "" + continue + elif keyname == 1: + if item.isspace(): + keyname = 2 + continue + elif item == ".": + dottedkey = True + continue + elif item.isalnum() or item == "_" or item == "-": + continue + elif dottedkey and sl[i - 1] == "." and (item == '"' or item == "'"): + openstring = True + openstrchar = item + continue + elif keyname == 2: + if item.isspace(): + if dottedkey: + nextitem = sl[i + 1] + if not nextitem.isspace() and nextitem != ".": + keyname = 1 + continue + if item == ".": + dottedkey = True + nextitem = sl[i + 1] + if not nextitem.isspace() and nextitem != ".": + keyname = 1 + continue + if item == "=": + keyname = 0 + prev_key = key[:-1].rstrip() + key = "" + dottedkey = False + else: + raise TomlDecodeError( + "Found invalid character in key name: '" + + item + + "'. Try quoting the key name.", + original, + i, + ) + if item == "'" and openstrchar != '"': + k = 1 + try: + while sl[i - k] == "'": + k += 1 + if k == 3: + break + except IndexError: + pass + if k == 3: + multilinestr = not multilinestr + openstring = multilinestr + else: + openstring = not openstring + if openstring: + openstrchar = "'" + else: + openstrchar = "" + if item == '"' and openstrchar != "'": + oddbackslash = False + k = 1 + tripquote = False + try: + while sl[i - k] == '"': + k += 1 + if k == 3: + tripquote = True + break + if k == 1 or (k == 3 and tripquote): + while sl[i - k] == "\\": + oddbackslash = not oddbackslash + k += 1 + except IndexError: + pass + if not oddbackslash: + if tripquote: + multilinestr = not multilinestr + openstring = multilinestr + else: + openstring = not openstring + if openstring: + openstrchar = '"' + else: + openstrchar = "" + if item == "#" and (not openstring and not keygroup and not arrayoftables): + j = i + comment = "" + try: + while sl[j] != "\n": + comment += s[j] + sl[j] = " " + j += 1 + except IndexError: + break + if not openarr: + decoder.preserve_comment(line_no, prev_key, comment, beginline) + if item == "[" and (not openstring and not keygroup and not arrayoftables): + if beginline: + if len(sl) > i + 1 and sl[i + 1] == "[": + arrayoftables = True + else: + keygroup = True + else: + openarr += 1 + if item == "]" and not openstring: + if keygroup: + keygroup = False + elif arrayoftables: + if sl[i - 1] == "]": + arrayoftables = False + else: + openarr -= 1 + if item == "\n": + if openstring or multilinestr: + if not multilinestr: + raise TomlDecodeError("Unbalanced quotes", original, i) + if (sl[i - 1] == "'" or sl[i - 1] == '"') and (sl[i - 2] == sl[i - 1]): + sl[i] = sl[i - 1] + if sl[i - 3] == sl[i - 1]: + sl[i - 3] = " " + elif openarr: + sl[i] = " " + else: + beginline = True + line_no += 1 + elif beginline and sl[i] != " " and sl[i] != "\t": + beginline = False + if not keygroup and not arrayoftables: + if sl[i] == "=": + raise TomlDecodeError("Found empty keyname. ", original, i) + keyname = 1 + key += item + if keyname: + raise TomlDecodeError( + "Key name found without value." " Reached end of file.", original, len(s) + ) + if openstring: # reached EOF and have an unterminated string + raise TomlDecodeError( + "Unterminated string found." " Reached end of file.", original, len(s) + ) + s = "".join(sl) + s = s.split("\n") + multikey = None + multilinestr = "" + multibackslash = False + pos = 0 + for idx, line in enumerate(s): + if idx > 0: + pos += len(s[idx - 1]) + 1 + + decoder.embed_comments(idx, currentlevel) + + if not multilinestr or multibackslash or "\n" not in multilinestr: + line = line.strip() + if line == "" and (not multikey or multibackslash): + continue + if multikey: + if multibackslash: + multilinestr += line + else: + multilinestr += line + multibackslash = False + closed = False + if multilinestr[0] == "[": + closed = line[-1] == "]" + elif len(line) > 2: + closed = ( + line[-1] == multilinestr[0] + and line[-2] == multilinestr[0] + and line[-3] == multilinestr[0] + ) + if closed: + try: + value, vtype = decoder.load_value(multilinestr) + except ValueError as err: + raise TomlDecodeError(str(err), original, pos) + currentlevel[multikey] = value + multikey = None + multilinestr = "" + else: + k = len(multilinestr) - 1 + while k > -1 and multilinestr[k] == "\\": + multibackslash = not multibackslash + k -= 1 + if multibackslash: + multilinestr = multilinestr[:-1] + else: + multilinestr += "\n" + continue + if line[0] == "[": + arrayoftables = False + if len(line) == 1: + raise TomlDecodeError( + "Opening key group bracket on line by " "itself.", original, pos + ) + if line[1] == "[": + arrayoftables = True + line = line[2:] + splitstr = "]]" + else: + line = line[1:] + splitstr = "]" + i = 1 + quotesplits = decoder._get_split_on_quotes(line) + quoted = False + for quotesplit in quotesplits: + if not quoted and splitstr in quotesplit: + break + i += quotesplit.count(splitstr) + quoted = not quoted + line = line.split(splitstr, i) + if len(line) < i + 1 or line[-1].strip() != "": + raise TomlDecodeError("Key group not on a line by itself.", original, pos) + groups = splitstr.join(line[:-1]).split(".") + i = 0 + while i < len(groups): + groups[i] = groups[i].strip() + if len(groups[i]) > 0 and (groups[i][0] == '"' or groups[i][0] == "'"): + groupstr = groups[i] + j = i + 1 + while not groupstr[0] == groupstr[-1]: + j += 1 + if j > len(groups) + 2: + raise TomlDecodeError( + "Invalid group name '" + groupstr + "' Something " + "went wrong.", + original, + pos, + ) + groupstr = ".".join(groups[i:j]).strip() + groups[i] = groupstr[1:-1] + groups[i + 1 : j] = [] + else: + if not _groupname_re.match(groups[i]): + raise TomlDecodeError( + "Invalid group name '" + groups[i] + "'. Try quoting it.", original, pos + ) + i += 1 + currentlevel = retval + for i in _range(len(groups)): + group = groups[i] + if group == "": + raise TomlDecodeError( + "Can't have a keygroup with an empty " "name", original, pos + ) + try: + currentlevel[group] + if i == len(groups) - 1: + if group in implicitgroups: + implicitgroups.remove(group) + if arrayoftables: + raise TomlDecodeError( + "An implicitly defined " "table can't be an array", + original, + pos, + ) + elif arrayoftables: + currentlevel[group].append(decoder.get_empty_table()) + else: + raise TomlDecodeError( + "What? " + group + " already exists?" + str(currentlevel), + original, + pos, + ) + except TypeError: + currentlevel = currentlevel[-1] + if group not in currentlevel: + currentlevel[group] = decoder.get_empty_table() + if i == len(groups) - 1 and arrayoftables: + currentlevel[group] = [decoder.get_empty_table()] + except KeyError: + if i != len(groups) - 1: + implicitgroups.append(group) + currentlevel[group] = decoder.get_empty_table() + if i == len(groups) - 1 and arrayoftables: + currentlevel[group] = [decoder.get_empty_table()] + currentlevel = currentlevel[group] + if arrayoftables: + try: + currentlevel = currentlevel[-1] + except KeyError: + pass + elif line[0] == "{": + if line[-1] != "}": + raise TomlDecodeError( + "Line breaks are not allowed in inline" "objects", original, pos + ) + try: + decoder.load_inline_object(line, currentlevel, multikey, multibackslash) + except ValueError as err: + raise TomlDecodeError(str(err), original, pos) + elif "=" in line: + try: + ret = decoder.load_line(line, currentlevel, multikey, multibackslash) + except ValueError as err: + raise TomlDecodeError(str(err), original, pos) + if ret is not None: + multikey, multilinestr, multibackslash = ret + return retval + + +def _load_date(val): + microsecond = 0 + tz = None + try: + if len(val) > 19: + if val[19] == ".": + if val[-1].upper() == "Z": + subsecondval = val[20:-1] + tzval = "Z" + else: + subsecondvalandtz = val[20:] + if "+" in subsecondvalandtz: + splitpoint = subsecondvalandtz.index("+") + subsecondval = subsecondvalandtz[:splitpoint] + tzval = subsecondvalandtz[splitpoint:] + elif "-" in subsecondvalandtz: + splitpoint = subsecondvalandtz.index("-") + subsecondval = subsecondvalandtz[:splitpoint] + tzval = subsecondvalandtz[splitpoint:] + else: + tzval = None + subsecondval = subsecondvalandtz + if tzval is not None: + tz = TomlTz(tzval) + microsecond = int(int(subsecondval) * (10 ** (6 - len(subsecondval)))) + else: + tz = TomlTz(val[19:]) + except ValueError: + tz = None + if "-" not in val[1:]: + return None + try: + if len(val) == 10: + d = datetime.date(int(val[:4]), int(val[5:7]), int(val[8:10])) + else: + d = datetime.datetime( + int(val[:4]), + int(val[5:7]), + int(val[8:10]), + int(val[11:13]), + int(val[14:16]), + int(val[17:19]), + microsecond, + tz, + ) + except ValueError: + return None + return d + + +def _load_unicode_escapes(v, hexbytes, prefix): + skip = False + i = len(v) - 1 + while i > -1 and v[i] == "\\": + skip = not skip + i -= 1 + for hx in hexbytes: + if skip: + skip = False + i = len(hx) - 1 + while i > -1 and hx[i] == "\\": + skip = not skip + i -= 1 + v += prefix + v += hx + continue + hxb = "" + i = 0 + hxblen = 4 + if prefix == "\\U": + hxblen = 8 + hxb = "".join(hx[i : i + hxblen]).lower() + if hxb.strip("0123456789abcdef"): + raise ValueError("Invalid escape sequence: " + hxb) + if hxb[0] == "d" and hxb[1].strip("01234567"): + raise ValueError( + "Invalid escape sequence: " + hxb + ". Only scalar unicode points are allowed." + ) + v += unichr(int(hxb, 16)) + v += unicode(hx[len(hxb) :]) + return v + + +# Unescape TOML string values. + +# content after the \ +_escapes = ["0", "b", "f", "n", "r", "t", '"'] +# What it should be replaced by +_escapedchars = ["\0", "\b", "\f", "\n", "\r", "\t", '"'] +# Used for substitution +_escape_to_escapedchars = dict(zip(_escapes, _escapedchars)) + + +def _unescape(v): + """Unescape characters in a TOML string.""" + i = 0 + backslash = False + while i < len(v): + if backslash: + backslash = False + if v[i] in _escapes: + v = v[: i - 1] + _escape_to_escapedchars[v[i]] + v[i + 1 :] + elif v[i] == "\\": + v = v[: i - 1] + v[i:] + elif v[i] == "u" or v[i] == "U": + i += 1 + else: + raise ValueError("Reserved escape sequence used") + continue + elif v[i] == "\\": + backslash = True + i += 1 + return v + + +class InlineTableDict(object): + """Sentinel subclass of dict for inline tables.""" + + +class TomlDecoder(object): + def __init__(self, _dict=dict): + self._dict = _dict + + def get_empty_table(self): + return self._dict() + + def get_empty_inline_table(self): + class DynamicInlineTableDict(self._dict, InlineTableDict): + """Concrete sentinel subclass for inline tables. + It is a subclass of _dict which is passed in dynamically at load + time + + It is also a subclass of InlineTableDict + """ + + return DynamicInlineTableDict() + + def load_inline_object(self, line, currentlevel, multikey=False, multibackslash=False): + candidate_groups = line[1:-1].split(",") + groups = [] + if len(candidate_groups) == 1 and not candidate_groups[0].strip(): + candidate_groups.pop() + while len(candidate_groups) > 0: + candidate_group = candidate_groups.pop(0) + try: + _, value = candidate_group.split("=", 1) + except ValueError: + raise ValueError("Invalid inline table encountered") + value = value.strip() + if (value[0] == value[-1] and value[0] in ('"', "'")) or ( + value[0] in "-0123456789" + or value in ("true", "false") + or (value[0] == "[" and value[-1] == "]") + or (value[0] == "{" and value[-1] == "}") + ): + groups.append(candidate_group) + elif len(candidate_groups) > 0: + candidate_groups[0] = candidate_group + "," + candidate_groups[0] + else: + raise ValueError("Invalid inline table value encountered") + for group in groups: + status = self.load_line(group, currentlevel, multikey, multibackslash) + if status is not None: + break + + def _get_split_on_quotes(self, line): + doublequotesplits = line.split('"') + quoted = False + quotesplits = [] + if len(doublequotesplits) > 1 and "'" in doublequotesplits[0]: + singlequotesplits = doublequotesplits[0].split("'") + doublequotesplits = doublequotesplits[1:] + while len(singlequotesplits) % 2 == 0 and len(doublequotesplits): + singlequotesplits[-1] += '"' + doublequotesplits[0] + doublequotesplits = doublequotesplits[1:] + if "'" in singlequotesplits[-1]: + singlequotesplits = singlequotesplits[:-1] + singlequotesplits[-1].split("'") + quotesplits += singlequotesplits + for doublequotesplit in doublequotesplits: + if quoted: + quotesplits.append(doublequotesplit) + else: + quotesplits += doublequotesplit.split("'") + quoted = not quoted + return quotesplits + + def load_line(self, line, currentlevel, multikey, multibackslash): + i = 1 + quotesplits = self._get_split_on_quotes(line) + quoted = False + for quotesplit in quotesplits: + if not quoted and "=" in quotesplit: + break + i += quotesplit.count("=") + quoted = not quoted + pair = line.split("=", i) + strictly_valid = _strictly_valid_num(pair[-1]) + if _number_with_underscores.match(pair[-1]): + pair[-1] = pair[-1].replace("_", "") + while len(pair[-1]) and ( + pair[-1][0] != " " + and pair[-1][0] != "\t" + and pair[-1][0] != "'" + and pair[-1][0] != '"' + and pair[-1][0] != "[" + and pair[-1][0] != "{" + and pair[-1].strip() != "true" + and pair[-1].strip() != "false" + ): + try: + float(pair[-1]) + break + except ValueError: + pass + if _load_date(pair[-1]) is not None: + break + if TIME_RE.match(pair[-1]): + break + i += 1 + prev_val = pair[-1] + pair = line.split("=", i) + if prev_val == pair[-1]: + raise ValueError("Invalid date or number") + if strictly_valid: + strictly_valid = _strictly_valid_num(pair[-1]) + pair = ["=".join(pair[:-1]).strip(), pair[-1].strip()] + if "." in pair[0]: + if '"' in pair[0] or "'" in pair[0]: + quotesplits = self._get_split_on_quotes(pair[0]) + quoted = False + levels = [] + for quotesplit in quotesplits: + if quoted: + levels.append(quotesplit) + else: + levels += [level.strip() for level in quotesplit.split(".")] + quoted = not quoted + else: + levels = pair[0].split(".") + while levels[-1] == "": + levels = levels[:-1] + for level in levels[:-1]: + if level == "": + continue + if level not in currentlevel: + currentlevel[level] = self.get_empty_table() + currentlevel = currentlevel[level] + pair[0] = levels[-1].strip() + elif (pair[0][0] == '"' or pair[0][0] == "'") and (pair[0][-1] == pair[0][0]): + pair[0] = _unescape(pair[0][1:-1]) + k, koffset = self._load_line_multiline_str(pair[1]) + if k > -1: + while k > -1 and pair[1][k + koffset] == "\\": + multibackslash = not multibackslash + k -= 1 + if multibackslash: + multilinestr = pair[1][:-1] + else: + multilinestr = pair[1] + "\n" + multikey = pair[0] + else: + value, vtype = self.load_value(pair[1], strictly_valid) + try: + currentlevel[pair[0]] + raise ValueError("Duplicate keys!") + except TypeError: + raise ValueError("Duplicate keys!") + except KeyError: + if multikey: + return multikey, multilinestr, multibackslash + else: + currentlevel[pair[0]] = value + + def _load_line_multiline_str(self, p): + poffset = 0 + if len(p) < 3: + return -1, poffset + if p[0] == "[" and (p.strip()[-1] != "]" and self._load_array_isstrarray(p)): + newp = p[1:].strip().split(",") + while len(newp) > 1 and newp[-1][0] != '"' and newp[-1][0] != "'": + newp = newp[:-2] + [newp[-2] + "," + newp[-1]] + newp = newp[-1] + poffset = len(p) - len(newp) + p = newp + if p[0] != '"' and p[0] != "'": + return -1, poffset + if p[1] != p[0] or p[2] != p[0]: + return -1, poffset + if len(p) > 5 and p[-1] == p[0] and p[-2] == p[0] and p[-3] == p[0]: + return -1, poffset + return len(p) - 1, poffset + + def load_value(self, v, strictly_valid=True): + if not v: + raise ValueError("Empty value is invalid") + if v == "true": + return (True, "bool") + elif v == "false": + return (False, "bool") + elif v[0] == '"' or v[0] == "'": + quotechar = v[0] + testv = v[1:].split(quotechar) + triplequote = False + triplequotecount = 0 + if len(testv) > 1 and testv[0] == "" and testv[1] == "": + testv = testv[2:] + triplequote = True + closed = False + for tv in testv: + if tv == "": + if triplequote: + triplequotecount += 1 + else: + closed = True + else: + oddbackslash = False + try: + i = -1 + j = tv[i] + while j == "\\": + oddbackslash = not oddbackslash + i -= 1 + j = tv[i] + except IndexError: + pass + if not oddbackslash: + if closed: + raise ValueError( + "Found tokens after a closed " + "string. Invalid TOML." + ) + else: + if not triplequote or triplequotecount > 1: + closed = True + else: + triplequotecount = 0 + if quotechar == '"': + escapeseqs = v.split("\\")[1:] + backslash = False + for i in escapeseqs: + if i == "": + backslash = not backslash + else: + if i[0] not in _escapes and (i[0] != "u" and i[0] != "U" and not backslash): + raise ValueError("Reserved escape sequence used") + if backslash: + backslash = False + for prefix in ["\\u", "\\U"]: + if prefix in v: + hexbytes = v.split(prefix) + v = _load_unicode_escapes(hexbytes[0], hexbytes[1:], prefix) + v = _unescape(v) + if len(v) > 1 and v[1] == quotechar and (len(v) < 3 or v[1] == v[2]): + v = v[2:-2] + return (v[1:-1], "str") + elif v[0] == "[": + return (self.load_array(v), "array") + elif v[0] == "{": + inline_object = self.get_empty_inline_table() + self.load_inline_object(v, inline_object) + return (inline_object, "inline_object") + elif TIME_RE.match(v): + h, m, s, _, ms = TIME_RE.match(v).groups() + time = datetime.time(int(h), int(m), int(s), int(ms) if ms else 0) + return (time, "time") + else: + parsed_date = _load_date(v) + if parsed_date is not None: + return (parsed_date, "date") + if not strictly_valid: + raise ValueError("Weirdness with leading zeroes or " "underscores in your number.") + itype = "int" + neg = False + if v[0] == "-": + neg = True + v = v[1:] + elif v[0] == "+": + v = v[1:] + v = v.replace("_", "") + lowerv = v.lower() + if "." in v or ("x" not in v and ("e" in v or "E" in v)): + if "." in v and v.split(".", 1)[1] == "": + raise ValueError("This float is missing digits after " "the point") + if v[0] not in "0123456789": + raise ValueError("This float doesn't have a leading " "digit") + v = float(v) + itype = "float" + elif len(lowerv) == 3 and (lowerv == "inf" or lowerv == "nan"): + v = float(v) + itype = "float" + if itype == "int": + v = int(v, 0) + if neg: + return (0 - v, itype) + return (v, itype) + + def bounded_string(self, s): + if len(s) == 0: + return True + if s[-1] != s[0]: + return False + i = -2 + backslash = False + while len(s) + i > 0: + if s[i] == "\\": + backslash = not backslash + i -= 1 + else: + break + return not backslash + + def _load_array_isstrarray(self, a): + a = a[1:-1].strip() + if a != "" and (a[0] == '"' or a[0] == "'"): + return True + return False + + def load_array(self, a): + atype = None + retval = [] + a = a.strip() + if "[" not in a[1:-1] or "" != a[1:-1].split("[")[0].strip(): + strarray = self._load_array_isstrarray(a) + if not a[1:-1].strip().startswith("{"): + a = a[1:-1].split(",") + else: + # a is an inline object, we must find the matching parenthesis + # to define groups + new_a = [] + start_group_index = 1 + end_group_index = 2 + open_bracket_count = 1 if a[start_group_index] == "{" else 0 + in_str = False + while end_group_index < len(a[1:]): + if a[end_group_index] == '"' or a[end_group_index] == "'": + if in_str: + backslash_index = end_group_index - 1 + while backslash_index > -1 and a[backslash_index] == "\\": + in_str = not in_str + backslash_index -= 1 + in_str = not in_str + if not in_str and a[end_group_index] == "{": + open_bracket_count += 1 + if in_str or a[end_group_index] != "}": + end_group_index += 1 + continue + elif a[end_group_index] == "}" and open_bracket_count > 1: + open_bracket_count -= 1 + end_group_index += 1 + continue + + # Increase end_group_index by 1 to get the closing bracket + end_group_index += 1 + + new_a.append(a[start_group_index:end_group_index]) + + # The next start index is at least after the closing + # bracket, a closing bracket can be followed by a comma + # since we are in an array. + start_group_index = end_group_index + 1 + while start_group_index < len(a[1:]) and a[start_group_index] != "{": + start_group_index += 1 + end_group_index = start_group_index + 1 + a = new_a + b = 0 + if strarray: + while b < len(a) - 1: + ab = a[b].strip() + while not self.bounded_string(ab) or ( + len(ab) > 2 + and ab[0] == ab[1] == ab[2] + and ab[-2] != ab[0] + and ab[-3] != ab[0] + ): + a[b] = a[b] + "," + a[b + 1] + ab = a[b].strip() + if b < len(a) - 2: + a = a[: b + 1] + a[b + 2 :] + else: + a = a[: b + 1] + b += 1 + else: + al = list(a[1:-1]) + a = [] + openarr = 0 + j = 0 + for i in _range(len(al)): + if al[i] == "[": + openarr += 1 + elif al[i] == "]": + openarr -= 1 + elif al[i] == "," and not openarr: + a.append("".join(al[j:i])) + j = i + 1 + a.append("".join(al[j:])) + for i in _range(len(a)): + a[i] = a[i].strip() + if a[i] != "": + nval, ntype = self.load_value(a[i]) + if atype: + if ntype != atype: + raise ValueError("Not a homogeneous array") + else: + atype = ntype + retval.append(nval) + return retval + + def preserve_comment(self, line_no, key, comment, beginline): + pass + + def embed_comments(self, idx, currentlevel): + pass + + +class TomlPreserveCommentDecoder(TomlDecoder): + def __init__(self, _dict=dict): + self.saved_comments = {} + super(TomlPreserveCommentDecoder, self).__init__(_dict) + + def preserve_comment(self, line_no, key, comment, beginline): + self.saved_comments[line_no] = (key, comment, beginline) + + def embed_comments(self, idx, currentlevel): + if idx not in self.saved_comments: + return + + key, comment, beginline = self.saved_comments[idx] + currentlevel[key] = CommentValue(currentlevel[key], comment, beginline, self._dict) diff --git a/WebCrawler/venv/Lib/site-packages/isort/_vendored/toml/encoder.py b/WebCrawler/venv/Lib/site-packages/isort/_vendored/toml/encoder.py new file mode 100644 index 0000000..68ec60f --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/isort/_vendored/toml/encoder.py @@ -0,0 +1,295 @@ +import datetime +import re +import sys +from decimal import Decimal + +from .decoder import InlineTableDict + +if sys.version_info >= (3,): + unicode = str + + +def dump(o, f, encoder=None): + """Writes out dict as toml to a file + + Args: + o: Object to dump into toml + f: File descriptor where the toml should be stored + encoder: The ``TomlEncoder`` to use for constructing the output string + + Returns: + String containing the toml corresponding to dictionary + + Raises: + TypeError: When anything other than file descriptor is passed + """ + + if not f.write: + raise TypeError("You can only dump an object to a file descriptor") + d = dumps(o, encoder=encoder) + f.write(d) + return d + + +def dumps(o, encoder=None): + """Stringifies input dict as toml + + Args: + o: Object to dump into toml + encoder: The ``TomlEncoder`` to use for constructing the output string + + Returns: + String containing the toml corresponding to dict + + Examples: + ```python + >>> import toml + >>> output = { + ... 'a': "I'm a string", + ... 'b': ["I'm", "a", "list"], + ... 'c': 2400 + ... } + >>> toml.dumps(output) + 'a = "I\'m a string"\nb = [ "I\'m", "a", "list",]\nc = 2400\n' + ``` + """ + + retval = "" + if encoder is None: + encoder = TomlEncoder(o.__class__) + addtoretval, sections = encoder.dump_sections(o, "") + retval += addtoretval + outer_objs = [id(o)] + while sections: + section_ids = [id(section) for section in sections] + for outer_obj in outer_objs: + if outer_obj in section_ids: + raise ValueError("Circular reference detected") + outer_objs += section_ids + newsections = encoder.get_empty_table() + for section in sections: + addtoretval, addtosections = encoder.dump_sections(sections[section], section) + + if addtoretval or (not addtoretval and not addtosections): + if retval and retval[-2:] != "\n\n": + retval += "\n" + retval += "[" + section + "]\n" + if addtoretval: + retval += addtoretval + for s in addtosections: + newsections[section + "." + s] = addtosections[s] + sections = newsections + return retval + + +def _dump_str(v): + if sys.version_info < (3,) and hasattr(v, "decode") and isinstance(v, str): + v = v.decode("utf-8") + v = "%r" % v + if v[0] == "u": + v = v[1:] + singlequote = v.startswith("'") + if singlequote or v.startswith('"'): + v = v[1:-1] + if singlequote: + v = v.replace("\\'", "'") + v = v.replace('"', '\\"') + v = v.split("\\x") + while len(v) > 1: + i = -1 + if not v[0]: + v = v[1:] + v[0] = v[0].replace("\\\\", "\\") + # No, I don't know why != works and == breaks + joinx = v[0][i] != "\\" + while v[0][:i] and v[0][i] == "\\": + joinx = not joinx + i -= 1 + if joinx: + joiner = "x" + else: + joiner = "u00" + v = [v[0] + joiner + v[1]] + v[2:] + return unicode('"' + v[0] + '"') + + +def _dump_float(v): + return "{}".format(v).replace("e+0", "e+").replace("e-0", "e-") + + +def _dump_time(v): + utcoffset = v.utcoffset() + if utcoffset is None: + return v.isoformat() + # The TOML norm specifies that it's local time thus we drop the offset + return v.isoformat()[:-6] + + +class TomlEncoder(object): + def __init__(self, _dict=dict, preserve=False): + self._dict = _dict + self.preserve = preserve + self.dump_funcs = { + str: _dump_str, + unicode: _dump_str, + list: self.dump_list, + bool: lambda v: unicode(v).lower(), + int: lambda v: v, + float: _dump_float, + Decimal: _dump_float, + datetime.datetime: lambda v: v.isoformat().replace("+00:00", "Z"), + datetime.time: _dump_time, + datetime.date: lambda v: v.isoformat(), + } + + def get_empty_table(self): + return self._dict() + + def dump_list(self, v): + retval = "[" + for u in v: + retval += " " + unicode(self.dump_value(u)) + "," + retval += "]" + return retval + + def dump_inline_table(self, section): + """Preserve inline table in its compact syntax instead of expanding + into subsection. + + https://github.com/toml-lang/toml#user-content-inline-table + """ + retval = "" + if isinstance(section, dict): + val_list = [] + for k, v in section.items(): + val = self.dump_inline_table(v) + val_list.append(k + " = " + val) + retval += "{ " + ", ".join(val_list) + " }\n" + return retval + else: + return unicode(self.dump_value(section)) + + def dump_value(self, v): + # Lookup function corresponding to v's type + dump_fn = self.dump_funcs.get(type(v)) + if dump_fn is None and hasattr(v, "__iter__"): + dump_fn = self.dump_funcs[list] + # Evaluate function (if it exists) else return v + return dump_fn(v) if dump_fn is not None else self.dump_funcs[str](v) + + def dump_sections(self, o, sup): + retstr = "" + if sup != "" and sup[-1] != ".": + sup += "." + retdict = self._dict() + arraystr = "" + for section in o: + section = unicode(section) + qsection = section + if not re.match(r"^[A-Za-z0-9_-]+$", section): + qsection = _dump_str(section) + if not isinstance(o[section], dict): + arrayoftables = False + if isinstance(o[section], list): + for a in o[section]: + if isinstance(a, dict): + arrayoftables = True + if arrayoftables: + for a in o[section]: + arraytabstr = "\n" + arraystr += "[[" + sup + qsection + "]]\n" + s, d = self.dump_sections(a, sup + qsection) + if s: + if s[0] == "[": + arraytabstr += s + else: + arraystr += s + while d: + newd = self._dict() + for dsec in d: + s1, d1 = self.dump_sections(d[dsec], sup + qsection + "." + dsec) + if s1: + arraytabstr += "[" + sup + qsection + "." + dsec + "]\n" + arraytabstr += s1 + for s1 in d1: + newd[dsec + "." + s1] = d1[s1] + d = newd + arraystr += arraytabstr + else: + if o[section] is not None: + retstr += qsection + " = " + unicode(self.dump_value(o[section])) + "\n" + elif self.preserve and isinstance(o[section], InlineTableDict): + retstr += qsection + " = " + self.dump_inline_table(o[section]) + else: + retdict[qsection] = o[section] + retstr += arraystr + return (retstr, retdict) + + +class TomlPreserveInlineDictEncoder(TomlEncoder): + def __init__(self, _dict=dict): + super(TomlPreserveInlineDictEncoder, self).__init__(_dict, True) + + +class TomlArraySeparatorEncoder(TomlEncoder): + def __init__(self, _dict=dict, preserve=False, separator=","): + super(TomlArraySeparatorEncoder, self).__init__(_dict, preserve) + if separator.strip() == "": + separator = "," + separator + elif separator.strip(" \t\n\r,"): + raise ValueError("Invalid separator for arrays") + self.separator = separator + + def dump_list(self, v): + t = [] + retval = "[" + for u in v: + t.append(self.dump_value(u)) + while t != []: + s = [] + for u in t: + if isinstance(u, list): + for r in u: + s.append(r) + else: + retval += " " + unicode(u) + self.separator + t = s + retval += "]" + return retval + + +class TomlNumpyEncoder(TomlEncoder): + def __init__(self, _dict=dict, preserve=False): + import numpy as np + + super(TomlNumpyEncoder, self).__init__(_dict, preserve) + self.dump_funcs[np.float16] = _dump_float + self.dump_funcs[np.float32] = _dump_float + self.dump_funcs[np.float64] = _dump_float + self.dump_funcs[np.int16] = self._dump_int + self.dump_funcs[np.int32] = self._dump_int + self.dump_funcs[np.int64] = self._dump_int + + def _dump_int(self, v): + return "{}".format(int(v)) + + +class TomlPreserveCommentEncoder(TomlEncoder): + def __init__(self, _dict=dict, preserve=False): + from toml.decoder import CommentValue + + super(TomlPreserveCommentEncoder, self).__init__(_dict, preserve) + self.dump_funcs[CommentValue] = lambda v: v.dump(self.dump_value) + + +class TomlPathlibEncoder(TomlEncoder): + def _dump_pathlib_path(self, v): + return _dump_str(str(v)) + + def dump_value(self, v): + if (3, 4) <= sys.version_info: + import pathlib + + if isinstance(v, pathlib.PurePath): + v = str(v) + return super(TomlPathlibEncoder, self).dump_value(v) diff --git a/WebCrawler/venv/Lib/site-packages/isort/_vendored/toml/ordered.py b/WebCrawler/venv/Lib/site-packages/isort/_vendored/toml/ordered.py new file mode 100644 index 0000000..013b31e --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/isort/_vendored/toml/ordered.py @@ -0,0 +1,13 @@ +from collections import OrderedDict + +from . import TomlDecoder, TomlEncoder + + +class TomlOrderedDecoder(TomlDecoder): + def __init__(self): + super(self.__class__, self).__init__(_dict=OrderedDict) + + +class TomlOrderedEncoder(TomlEncoder): + def __init__(self): + super(self.__class__, self).__init__(_dict=OrderedDict) diff --git a/WebCrawler/venv/Lib/site-packages/isort/_vendored/toml/tz.py b/WebCrawler/venv/Lib/site-packages/isort/_vendored/toml/tz.py new file mode 100644 index 0000000..46214bd --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/isort/_vendored/toml/tz.py @@ -0,0 +1,21 @@ +from datetime import timedelta, tzinfo + + +class TomlTz(tzinfo): + def __init__(self, toml_offset): + if toml_offset == "Z": + self._raw_offset = "+00:00" + else: + self._raw_offset = toml_offset + self._sign = -1 if self._raw_offset[0] == "-" else 1 + self._hours = int(self._raw_offset[1:3]) + self._minutes = int(self._raw_offset[4:6]) + + def tzname(self, dt): + return "UTC" + self._raw_offset + + def utcoffset(self, dt): + return self._sign * timedelta(hours=self._hours, minutes=self._minutes) + + def dst(self, dt): + return timedelta(0) diff --git a/WebCrawler/venv/Lib/site-packages/isort/_version.py b/WebCrawler/venv/Lib/site-packages/isort/_version.py new file mode 100644 index 0000000..f0b716b --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/isort/_version.py @@ -0,0 +1 @@ +__version__ = "5.6.4" diff --git a/WebCrawler/venv/Lib/site-packages/isort/api.py b/WebCrawler/venv/Lib/site-packages/isort/api.py new file mode 100644 index 0000000..5a2df6a --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/isort/api.py @@ -0,0 +1,389 @@ +import shutil +import sys +from io import StringIO +from pathlib import Path +from typing import Optional, TextIO, Union, cast +from warnings import warn + +from isort import core + +from . import io +from .exceptions import ( + ExistingSyntaxErrors, + FileSkipComment, + FileSkipSetting, + IntroducedSyntaxErrors, +) +from .format import ask_whether_to_apply_changes_to_file, create_terminal_printer, show_unified_diff +from .io import Empty +from .place import module as place_module # noqa: F401 +from .place import module_with_reason as place_module_with_reason # noqa: F401 +from .settings import DEFAULT_CONFIG, Config + + +def sort_code_string( + code: str, + extension: Optional[str] = None, + config: Config = DEFAULT_CONFIG, + file_path: Optional[Path] = None, + disregard_skip: bool = False, + show_diff: Union[bool, TextIO] = False, + **config_kwargs, +): + """Sorts any imports within the provided code string, returning a new string with them sorted. + + - **code**: The string of code with imports that need to be sorted. + - **extension**: The file extension that contains imports. Defaults to filename extension or py. + - **config**: The config object to use when sorting imports. + - **file_path**: The disk location where the code string was pulled from. + - **disregard_skip**: set to `True` if you want to ignore a skip set in config for this file. + - **show_diff**: If `True` the changes that need to be done will be printed to stdout, if a + TextIO stream is provided results will be written to it, otherwise no diff will be computed. + - ****config_kwargs**: Any config modifications. + """ + input_stream = StringIO(code) + output_stream = StringIO() + config = _config(path=file_path, config=config, **config_kwargs) + sort_stream( + input_stream, + output_stream, + extension=extension, + config=config, + file_path=file_path, + disregard_skip=disregard_skip, + show_diff=show_diff, + ) + output_stream.seek(0) + return output_stream.read() + + +def check_code_string( + code: str, + show_diff: Union[bool, TextIO] = False, + extension: Optional[str] = None, + config: Config = DEFAULT_CONFIG, + file_path: Optional[Path] = None, + disregard_skip: bool = False, + **config_kwargs, +) -> bool: + """Checks the order, format, and categorization of imports within the provided code string. + Returns `True` if everything is correct, otherwise `False`. + + - **code**: The string of code with imports that need to be sorted. + - **show_diff**: If `True` the changes that need to be done will be printed to stdout, if a + TextIO stream is provided results will be written to it, otherwise no diff will be computed. + - **extension**: The file extension that contains imports. Defaults to filename extension or py. + - **config**: The config object to use when sorting imports. + - **file_path**: The disk location where the code string was pulled from. + - **disregard_skip**: set to `True` if you want to ignore a skip set in config for this file. + - ****config_kwargs**: Any config modifications. + """ + config = _config(path=file_path, config=config, **config_kwargs) + return check_stream( + StringIO(code), + show_diff=show_diff, + extension=extension, + config=config, + file_path=file_path, + disregard_skip=disregard_skip, + ) + + +def sort_stream( + input_stream: TextIO, + output_stream: TextIO, + extension: Optional[str] = None, + config: Config = DEFAULT_CONFIG, + file_path: Optional[Path] = None, + disregard_skip: bool = False, + show_diff: Union[bool, TextIO] = False, + **config_kwargs, +) -> bool: + """Sorts any imports within the provided code stream, outputs to the provided output stream. + Returns `True` if anything is modified from the original input stream, otherwise `False`. + + - **input_stream**: The stream of code with imports that need to be sorted. + - **output_stream**: The stream where sorted imports should be written to. + - **extension**: The file extension that contains imports. Defaults to filename extension or py. + - **config**: The config object to use when sorting imports. + - **file_path**: The disk location where the code string was pulled from. + - **disregard_skip**: set to `True` if you want to ignore a skip set in config for this file. + - **show_diff**: If `True` the changes that need to be done will be printed to stdout, if a + TextIO stream is provided results will be written to it, otherwise no diff will be computed. + - ****config_kwargs**: Any config modifications. + """ + if show_diff: + _output_stream = StringIO() + _input_stream = StringIO(input_stream.read()) + changed = sort_stream( + input_stream=_input_stream, + output_stream=_output_stream, + extension=extension, + config=config, + file_path=file_path, + disregard_skip=disregard_skip, + **config_kwargs, + ) + _output_stream.seek(0) + _input_stream.seek(0) + show_unified_diff( + file_input=_input_stream.read(), + file_output=_output_stream.read(), + file_path=file_path, + output=output_stream if show_diff is True else cast(TextIO, show_diff), + color_output=config.color_output, + ) + return changed + + config = _config(path=file_path, config=config, **config_kwargs) + content_source = str(file_path or "Passed in content") + if not disregard_skip: + if file_path and config.is_skipped(file_path): + raise FileSkipSetting(content_source) + + _internal_output = output_stream + + if config.atomic: + try: + file_content = input_stream.read() + compile(file_content, content_source, "exec", 0, 1) + input_stream = StringIO(file_content) + except SyntaxError: + raise ExistingSyntaxErrors(content_source) + + if not output_stream.readable(): + _internal_output = StringIO() + + try: + changed = core.process( + input_stream, + _internal_output, + extension=extension or (file_path and file_path.suffix.lstrip(".")) or "py", + config=config, + ) + except FileSkipComment: + raise FileSkipComment(content_source) + + if config.atomic: + _internal_output.seek(0) + try: + compile(_internal_output.read(), content_source, "exec", 0, 1) + _internal_output.seek(0) + if _internal_output != output_stream: + output_stream.write(_internal_output.read()) + except SyntaxError: # pragma: no cover + raise IntroducedSyntaxErrors(content_source) + + return changed + + +def check_stream( + input_stream: TextIO, + show_diff: Union[bool, TextIO] = False, + extension: Optional[str] = None, + config: Config = DEFAULT_CONFIG, + file_path: Optional[Path] = None, + disregard_skip: bool = False, + **config_kwargs, +) -> bool: + """Checks any imports within the provided code stream, returning `False` if any unsorted or + incorrectly imports are found or `True` if no problems are identified. + + - **input_stream**: The stream of code with imports that need to be sorted. + - **show_diff**: If `True` the changes that need to be done will be printed to stdout, if a + TextIO stream is provided results will be written to it, otherwise no diff will be computed. + - **extension**: The file extension that contains imports. Defaults to filename extension or py. + - **config**: The config object to use when sorting imports. + - **file_path**: The disk location where the code string was pulled from. + - **disregard_skip**: set to `True` if you want to ignore a skip set in config for this file. + - ****config_kwargs**: Any config modifications. + """ + config = _config(path=file_path, config=config, **config_kwargs) + + if show_diff: + input_stream = StringIO(input_stream.read()) + + changed: bool = sort_stream( + input_stream=input_stream, + output_stream=Empty, + extension=extension, + config=config, + file_path=file_path, + disregard_skip=disregard_skip, + ) + printer = create_terminal_printer(color=config.color_output) + if not changed: + if config.verbose and not config.only_modified: + printer.success(f"{file_path or ''} Everything Looks Good!") + return True + else: + printer.error(f"{file_path or ''} Imports are incorrectly sorted and/or formatted.") + if show_diff: + output_stream = StringIO() + input_stream.seek(0) + file_contents = input_stream.read() + sort_stream( + input_stream=StringIO(file_contents), + output_stream=output_stream, + extension=extension, + config=config, + file_path=file_path, + disregard_skip=disregard_skip, + ) + output_stream.seek(0) + + show_unified_diff( + file_input=file_contents, + file_output=output_stream.read(), + file_path=file_path, + output=None if show_diff is True else cast(TextIO, show_diff), + color_output=config.color_output, + ) + return False + + +def check_file( + filename: Union[str, Path], + show_diff: Union[bool, TextIO] = False, + config: Config = DEFAULT_CONFIG, + file_path: Optional[Path] = None, + disregard_skip: bool = True, + extension: Optional[str] = None, + **config_kwargs, +) -> bool: + """Checks any imports within the provided file, returning `False` if any unsorted or + incorrectly imports are found or `True` if no problems are identified. + + - **filename**: The name or Path of the file to check. + - **show_diff**: If `True` the changes that need to be done will be printed to stdout, if a + TextIO stream is provided results will be written to it, otherwise no diff will be computed. + - **config**: The config object to use when sorting imports. + - **file_path**: The disk location where the code string was pulled from. + - **disregard_skip**: set to `True` if you want to ignore a skip set in config for this file. + - **extension**: The file extension that contains imports. Defaults to filename extension or py. + - ****config_kwargs**: Any config modifications. + """ + with io.File.read(filename) as source_file: + return check_stream( + source_file.stream, + show_diff=show_diff, + extension=extension, + config=config, + file_path=file_path or source_file.path, + disregard_skip=disregard_skip, + **config_kwargs, + ) + + +def sort_file( + filename: Union[str, Path], + extension: Optional[str] = None, + config: Config = DEFAULT_CONFIG, + file_path: Optional[Path] = None, + disregard_skip: bool = True, + ask_to_apply: bool = False, + show_diff: Union[bool, TextIO] = False, + write_to_stdout: bool = False, + **config_kwargs, +) -> bool: + """Sorts and formats any groups of imports imports within the provided file or Path. + Returns `True` if the file has been changed, otherwise `False`. + + - **filename**: The name or Path of the file to format. + - **extension**: The file extension that contains imports. Defaults to filename extension or py. + - **config**: The config object to use when sorting imports. + - **file_path**: The disk location where the code string was pulled from. + - **disregard_skip**: set to `True` if you want to ignore a skip set in config for this file. + - **ask_to_apply**: If `True`, prompt before applying any changes. + - **show_diff**: If `True` the changes that need to be done will be printed to stdout, if a + TextIO stream is provided results will be written to it, otherwise no diff will be computed. + - **write_to_stdout**: If `True`, write to stdout instead of the input file. + - ****config_kwargs**: Any config modifications. + """ + with io.File.read(filename) as source_file: + actual_file_path = file_path or source_file.path + config = _config(path=actual_file_path, config=config, **config_kwargs) + changed: bool = False + try: + if write_to_stdout: + changed = sort_stream( + input_stream=source_file.stream, + output_stream=sys.stdout, + config=config, + file_path=actual_file_path, + disregard_skip=disregard_skip, + extension=extension, + ) + else: + tmp_file = source_file.path.with_suffix(source_file.path.suffix + ".isorted") + try: + with tmp_file.open( + "w", encoding=source_file.encoding, newline="" + ) as output_stream: + shutil.copymode(filename, tmp_file) + changed = sort_stream( + input_stream=source_file.stream, + output_stream=output_stream, + config=config, + file_path=actual_file_path, + disregard_skip=disregard_skip, + extension=extension, + ) + if changed: + if show_diff or ask_to_apply: + source_file.stream.seek(0) + with tmp_file.open( + encoding=source_file.encoding, newline="" + ) as tmp_out: + show_unified_diff( + file_input=source_file.stream.read(), + file_output=tmp_out.read(), + file_path=actual_file_path, + output=None if show_diff is True else cast(TextIO, show_diff), + color_output=config.color_output, + ) + if show_diff or ( + ask_to_apply + and not ask_whether_to_apply_changes_to_file( + str(source_file.path) + ) + ): + return False + source_file.stream.close() + tmp_file.replace(source_file.path) + if not config.quiet: + print(f"Fixing {source_file.path}") + finally: + try: # Python 3.8+: use `missing_ok=True` instead of try except. + tmp_file.unlink() + except FileNotFoundError: + pass # pragma: no cover + except ExistingSyntaxErrors: + warn(f"{actual_file_path} unable to sort due to existing syntax errors") + except IntroducedSyntaxErrors: # pragma: no cover + warn(f"{actual_file_path} unable to sort as isort introduces new syntax errors") + + return changed + + +def _config( + path: Optional[Path] = None, config: Config = DEFAULT_CONFIG, **config_kwargs +) -> Config: + if path: + if ( + config is DEFAULT_CONFIG + and "settings_path" not in config_kwargs + and "settings_file" not in config_kwargs + ): + config_kwargs["settings_path"] = path + + if config_kwargs: + if config is not DEFAULT_CONFIG: + raise ValueError( + "You can either specify custom configuration options using kwargs or " + "passing in a Config object. Not Both!" + ) + + config = Config(**config_kwargs) + + return config diff --git a/WebCrawler/venv/Lib/site-packages/isort/comments.py b/WebCrawler/venv/Lib/site-packages/isort/comments.py new file mode 100644 index 0000000..b865b32 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/isort/comments.py @@ -0,0 +1,32 @@ +from typing import List, Optional, Tuple + + +def parse(line: str) -> Tuple[str, str]: + """Parses import lines for comments and returns back the + import statement and the associated comment. + """ + comment_start = line.find("#") + if comment_start != -1: + return (line[:comment_start], line[comment_start + 1 :].strip()) + + return (line, "") + + +def add_to_line( + comments: Optional[List[str]], + original_string: str = "", + removed: bool = False, + comment_prefix: str = "", +) -> str: + """Returns a string with comments added if removed is not set.""" + if removed: + return parse(original_string)[0] + + if not comments: + return original_string + else: + unique_comments: List[str] = [] + for comment in comments: + if comment not in unique_comments: + unique_comments.append(comment) + return f"{parse(original_string)[0]}{comment_prefix} {'; '.join(unique_comments)}" diff --git a/WebCrawler/venv/Lib/site-packages/isort/core.py b/WebCrawler/venv/Lib/site-packages/isort/core.py new file mode 100644 index 0000000..292bdc1 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/isort/core.py @@ -0,0 +1,420 @@ +import textwrap +from io import StringIO +from itertools import chain +from typing import List, TextIO, Union + +import isort.literal +from isort.settings import DEFAULT_CONFIG, Config + +from . import output, parse +from .exceptions import FileSkipComment +from .format import format_natural, remove_whitespace +from .settings import FILE_SKIP_COMMENTS + +CIMPORT_IDENTIFIERS = ("cimport ", "cimport*", "from.cimport") +IMPORT_START_IDENTIFIERS = ("from ", "from.import", "import ", "import*") + CIMPORT_IDENTIFIERS +COMMENT_INDICATORS = ('"""', "'''", "'", '"', "#") +CODE_SORT_COMMENTS = ( + "# isort: list", + "# isort: dict", + "# isort: set", + "# isort: unique-list", + "# isort: tuple", + "# isort: unique-tuple", + "# isort: assignments", +) + + +def process( + input_stream: TextIO, + output_stream: TextIO, + extension: str = "py", + config: Config = DEFAULT_CONFIG, +) -> bool: + """Parses stream identifying sections of contiguous imports and sorting them + + Code with unsorted imports is read from the provided `input_stream`, sorted and then + outputted to the specified `output_stream`. + + - `input_stream`: Text stream with unsorted import sections. + - `output_stream`: Text stream to output sorted inputs into. + - `config`: Config settings to use when sorting imports. Defaults settings. + - *Default*: `isort.settings.DEFAULT_CONFIG`. + - `extension`: The file extension or file extension rules that should be used. + - *Default*: `"py"`. + - *Choices*: `["py", "pyi", "pyx"]`. + + Returns `True` if there were changes that needed to be made (errors present) from what + was provided in the input_stream, otherwise `False`. + """ + line_separator: str = config.line_ending + add_imports: List[str] = [format_natural(addition) for addition in config.add_imports] + import_section: str = "" + next_import_section: str = "" + next_cimports: bool = False + in_quote: str = "" + first_comment_index_start: int = -1 + first_comment_index_end: int = -1 + contains_imports: bool = False + in_top_comment: bool = False + first_import_section: bool = True + indent: str = "" + isort_off: bool = False + code_sorting: Union[bool, str] = False + code_sorting_section: str = "" + code_sorting_indent: str = "" + cimports: bool = False + made_changes: bool = False + stripped_line: str = "" + end_of_file: bool = False + verbose_output: List[str] = [] + + if config.float_to_top: + new_input = "" + current = "" + isort_off = False + for line in chain(input_stream, (None,)): + if isort_off and line is not None: + if line == "# isort: on\n": + isort_off = False + new_input += line + elif line in ("# isort: split\n", "# isort: off\n", None) or str(line).endswith( + "# isort: split\n" + ): + if line == "# isort: off\n": + isort_off = True + if current: + if add_imports: + add_line_separator = line_separator or "\n" + current += add_line_separator + add_line_separator.join(add_imports) + add_imports = [] + parsed = parse.file_contents(current, config=config) + verbose_output += parsed.verbose_output + extra_space = "" + while current and current[-1] == "\n": + extra_space += "\n" + current = current[:-1] + extra_space = extra_space.replace("\n", "", 1) + sorted_output = output.sorted_imports( + parsed, config, extension, import_type="import" + ) + made_changes = made_changes or _has_changed( + before=current, + after=sorted_output, + line_separator=parsed.line_separator, + ignore_whitespace=config.ignore_whitespace, + ) + new_input += sorted_output + new_input += extra_space + current = "" + new_input += line or "" + else: + current += line or "" + + input_stream = StringIO(new_input) + + for index, line in enumerate(chain(input_stream, (None,))): + if line is None: + if index == 0 and not config.force_adds: + return False + + not_imports = True + end_of_file = True + line = "" + if not line_separator: + line_separator = "\n" + + if code_sorting and code_sorting_section: + output_stream.write( + textwrap.indent( + isort.literal.assignment( + code_sorting_section, + str(code_sorting), + extension, + config=_indented_config(config, indent), + ), + code_sorting_indent, + ) + ) + else: + stripped_line = line.strip() + if stripped_line and not line_separator: + line_separator = line[len(line.rstrip()) :].replace(" ", "").replace("\t", "") + + for file_skip_comment in FILE_SKIP_COMMENTS: + if file_skip_comment in line: + raise FileSkipComment("Passed in content") + + if not in_quote and stripped_line == "# isort: off": + isort_off = True + + if ( + (index == 0 or (index in (1, 2) and not contains_imports)) + and stripped_line.startswith("#") + and stripped_line not in config.section_comments + ): + in_top_comment = True + elif in_top_comment: + if not line.startswith("#") or stripped_line in config.section_comments: + in_top_comment = False + first_comment_index_end = index - 1 + + was_in_quote = bool(in_quote) + if (not stripped_line.startswith("#") or in_quote) and '"' in line or "'" in line: + char_index = 0 + if first_comment_index_start == -1 and ( + line.startswith('"') or line.startswith("'") + ): + first_comment_index_start = index + while char_index < len(line): + if line[char_index] == "\\": + char_index += 1 + elif in_quote: + if line[char_index : char_index + len(in_quote)] == in_quote: + in_quote = "" + if first_comment_index_end < first_comment_index_start: + first_comment_index_end = index + elif line[char_index] in ("'", '"'): + long_quote = line[char_index : char_index + 3] + if long_quote in ('"""', "'''"): + in_quote = long_quote + char_index += 2 + else: + in_quote = line[char_index] + elif line[char_index] == "#": + break + char_index += 1 + + not_imports = bool(in_quote) or was_in_quote or in_top_comment or isort_off + if not (in_quote or was_in_quote or in_top_comment): + if isort_off: + if stripped_line == "# isort: on": + isort_off = False + elif stripped_line.endswith("# isort: split"): + not_imports = True + elif stripped_line in CODE_SORT_COMMENTS: + code_sorting = stripped_line.split("isort: ")[1].strip() + code_sorting_indent = line[: -len(line.lstrip())] + not_imports = True + elif code_sorting: + if not stripped_line: + output_stream.write( + textwrap.indent( + isort.literal.assignment( + code_sorting_section, + str(code_sorting), + extension, + config=_indented_config(config, indent), + ), + code_sorting_indent, + ) + ) + not_imports = True + code_sorting = False + code_sorting_section = "" + code_sorting_indent = "" + else: + code_sorting_section += line + line = "" + elif stripped_line in config.section_comments: + if import_section and not contains_imports: + output_stream.write(import_section) + import_section = line + not_imports = False + else: + import_section += line + indent = line[: -len(line.lstrip())] + elif not (stripped_line or contains_imports): + not_imports = True + elif ( + not stripped_line + or stripped_line.startswith("#") + and (not indent or indent + line.lstrip() == line) + and not config.treat_all_comments_as_code + and stripped_line not in config.treat_comments_as_code + ): + import_section += line + elif stripped_line.startswith(IMPORT_START_IDENTIFIERS): + contains_imports = True + + new_indent = line[: -len(line.lstrip())] + import_statement = line + stripped_line = line.strip().split("#")[0] + while stripped_line.endswith("\\") or ( + "(" in stripped_line and ")" not in stripped_line + ): + if stripped_line.endswith("\\"): + while stripped_line and stripped_line.endswith("\\"): + line = input_stream.readline() + stripped_line = line.strip().split("#")[0] + import_statement += line + else: + while ")" not in stripped_line: + line = input_stream.readline() + stripped_line = line.strip().split("#")[0] + import_statement += line + + cimport_statement: bool = False + if ( + import_statement.lstrip().startswith(CIMPORT_IDENTIFIERS) + or " cimport " in import_statement + or " cimport*" in import_statement + or " cimport(" in import_statement + or ".cimport" in import_statement + ): + cimport_statement = True + + if cimport_statement != cimports or (new_indent != indent and import_section): + if import_section: + next_cimports = cimport_statement + next_import_section = import_statement + import_statement = "" + not_imports = True + line = "" + else: + cimports = cimport_statement + + indent = new_indent + import_section += import_statement + else: + not_imports = True + + if not_imports: + raw_import_section: str = import_section + if ( + add_imports + and (stripped_line or end_of_file) + and not config.append_only + and not in_top_comment + and not in_quote + and not import_section + and not line.lstrip().startswith(COMMENT_INDICATORS) + ): + import_section = line_separator.join(add_imports) + line_separator + if end_of_file and index != 0: + output_stream.write(line_separator) + contains_imports = True + add_imports = [] + + if next_import_section and not import_section: # pragma: no cover + raw_import_section = import_section = next_import_section + next_import_section = "" + + if import_section: + if add_imports and not indent: + import_section = ( + line_separator.join(add_imports) + line_separator + import_section + ) + contains_imports = True + add_imports = [] + + if not indent: + import_section += line + raw_import_section += line + if not contains_imports: + output_stream.write(import_section) + + else: + leading_whitespace = import_section[: -len(import_section.lstrip())] + trailing_whitespace = import_section[len(import_section.rstrip()) :] + if first_import_section and not import_section.lstrip( + line_separator + ).startswith(COMMENT_INDICATORS): + import_section = import_section.lstrip(line_separator) + raw_import_section = raw_import_section.lstrip(line_separator) + first_import_section = False + + if indent: + import_section = "".join( + line[len(indent) :] for line in import_section.splitlines(keepends=True) + ) + + parsed_content = parse.file_contents(import_section, config=config) + verbose_output += parsed_content.verbose_output + + sorted_import_section = output.sorted_imports( + parsed_content, + _indented_config(config, indent), + extension, + import_type="cimport" if cimports else "import", + ) + if not (import_section.strip() and not sorted_import_section): + if indent: + sorted_import_section = ( + leading_whitespace + + textwrap.indent(sorted_import_section, indent).strip() + + trailing_whitespace + ) + + made_changes = made_changes or _has_changed( + before=raw_import_section, + after=sorted_import_section, + line_separator=line_separator, + ignore_whitespace=config.ignore_whitespace, + ) + output_stream.write(sorted_import_section) + if not line and not indent and next_import_section: + output_stream.write(line_separator) + + if indent: + output_stream.write(line) + if not next_import_section: + indent = "" + + if next_import_section: + cimports = next_cimports + contains_imports = True + else: + contains_imports = False + import_section = next_import_section + next_import_section = "" + else: + output_stream.write(line) + not_imports = False + + if stripped_line and not in_quote and not import_section and not next_import_section: + if stripped_line == "yield": + while not stripped_line or stripped_line == "yield": + new_line = input_stream.readline() + if not new_line: + break + + output_stream.write(new_line) + stripped_line = new_line.strip().split("#")[0] + + if stripped_line.startswith("raise") or stripped_line.startswith("yield"): + while stripped_line.endswith("\\"): + new_line = input_stream.readline() + if not new_line: + break + + output_stream.write(new_line) + stripped_line = new_line.strip().split("#")[0] + + if made_changes and config.only_modified: + for output_str in verbose_output: + print(output_str) + + return made_changes + + +def _indented_config(config: Config, indent: str): + if not indent: + return config + + return Config( + config=config, + line_length=max(config.line_length - len(indent), 0), + wrap_length=max(config.wrap_length - len(indent), 0), + lines_after_imports=1, + ) + + +def _has_changed(before: str, after: str, line_separator: str, ignore_whitespace: bool) -> bool: + if ignore_whitespace: + return ( + remove_whitespace(before, line_separator=line_separator).strip() + != remove_whitespace(after, line_separator=line_separator).strip() + ) + else: + return before.strip() != after.strip() diff --git a/WebCrawler/venv/Lib/site-packages/isort/deprecated/__init__.py b/WebCrawler/venv/Lib/site-packages/isort/deprecated/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/WebCrawler/venv/Lib/site-packages/isort/deprecated/__pycache__/__init__.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/isort/deprecated/__pycache__/__init__.cpython-39.pyc new file mode 100644 index 0000000..8659ab5 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/isort/deprecated/__pycache__/__init__.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/isort/deprecated/__pycache__/finders.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/isort/deprecated/__pycache__/finders.cpython-39.pyc new file mode 100644 index 0000000..088fa67 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/isort/deprecated/__pycache__/finders.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/isort/deprecated/finders.py b/WebCrawler/venv/Lib/site-packages/isort/deprecated/finders.py new file mode 100644 index 0000000..dbb6fec --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/isort/deprecated/finders.py @@ -0,0 +1,415 @@ +"""Finders try to find right section for passed module name""" +import importlib.machinery +import inspect +import os +import os.path +import re +import sys +import sysconfig +from abc import ABCMeta, abstractmethod +from contextlib import contextmanager +from fnmatch import fnmatch +from functools import lru_cache +from glob import glob +from pathlib import Path +from typing import Dict, Iterable, Iterator, List, Optional, Pattern, Sequence, Tuple, Type + +from isort import sections +from isort.settings import KNOWN_SECTION_MAPPING, Config +from isort.utils import exists_case_sensitive + +try: + from pipreqs import pipreqs + +except ImportError: + pipreqs = None + +try: + from pip_api import parse_requirements + +except ImportError: + parse_requirements = None + +try: + from requirementslib import Pipfile + +except ImportError: + Pipfile = None + + +@contextmanager +def chdir(path: str) -> Iterator[None]: + """Context manager for changing dir and restoring previous workdir after exit.""" + curdir = os.getcwd() + os.chdir(path) + try: + yield + finally: + os.chdir(curdir) + + +class BaseFinder(metaclass=ABCMeta): + def __init__(self, config: Config) -> None: + self.config = config + + @abstractmethod + def find(self, module_name: str) -> Optional[str]: + raise NotImplementedError + + +class ForcedSeparateFinder(BaseFinder): + def find(self, module_name: str) -> Optional[str]: + for forced_separate in self.config.forced_separate: + # Ensure all forced_separate patterns will match to end of string + path_glob = forced_separate + if not forced_separate.endswith("*"): + path_glob = "%s*" % forced_separate + + if fnmatch(module_name, path_glob) or fnmatch(module_name, "." + path_glob): + return forced_separate + return None + + +class LocalFinder(BaseFinder): + def find(self, module_name: str) -> Optional[str]: + if module_name.startswith("."): + return "LOCALFOLDER" + return None + + +class KnownPatternFinder(BaseFinder): + def __init__(self, config: Config) -> None: + super().__init__(config) + + self.known_patterns: List[Tuple[Pattern[str], str]] = [] + for placement in reversed(config.sections): + known_placement = KNOWN_SECTION_MAPPING.get(placement, placement).lower() + config_key = f"known_{known_placement}" + known_patterns = list( + getattr(self.config, config_key, self.config.known_other.get(known_placement, [])) + ) + known_patterns = [ + pattern + for known_pattern in known_patterns + for pattern in self._parse_known_pattern(known_pattern) + ] + for known_pattern in known_patterns: + regexp = "^" + known_pattern.replace("*", ".*").replace("?", ".?") + "$" + self.known_patterns.append((re.compile(regexp), placement)) + + def _parse_known_pattern(self, pattern: str) -> List[str]: + """Expand pattern if identified as a directory and return found sub packages""" + if pattern.endswith(os.path.sep): + patterns = [ + filename + for filename in os.listdir(os.path.join(self.config.directory, pattern)) + if os.path.isdir(os.path.join(self.config.directory, pattern, filename)) + ] + else: + patterns = [pattern] + + return patterns + + def find(self, module_name: str) -> Optional[str]: + # Try to find most specific placement instruction match (if any) + parts = module_name.split(".") + module_names_to_check = (".".join(parts[:first_k]) for first_k in range(len(parts), 0, -1)) + for module_name_to_check in module_names_to_check: + for pattern, placement in self.known_patterns: + if pattern.match(module_name_to_check): + return placement + return None + + +class PathFinder(BaseFinder): + def __init__(self, config: Config, path: str = ".") -> None: + super().__init__(config) + + # restore the original import path (i.e. not the path to bin/isort) + root_dir = os.path.abspath(path) + src_dir = f"{root_dir}/src" + self.paths = [root_dir, src_dir] + + # virtual env + self.virtual_env = self.config.virtual_env or os.environ.get("VIRTUAL_ENV") + if self.virtual_env: + self.virtual_env = os.path.realpath(self.virtual_env) + self.virtual_env_src = "" + if self.virtual_env: + self.virtual_env_src = f"{self.virtual_env}/src/" + for venv_path in glob(f"{self.virtual_env}/lib/python*/site-packages"): + if venv_path not in self.paths: + self.paths.append(venv_path) + for nested_venv_path in glob(f"{self.virtual_env}/lib/python*/*/site-packages"): + if nested_venv_path not in self.paths: + self.paths.append(nested_venv_path) + for venv_src_path in glob(f"{self.virtual_env}/src/*"): + if os.path.isdir(venv_src_path): + self.paths.append(venv_src_path) + + # conda + self.conda_env = self.config.conda_env or os.environ.get("CONDA_PREFIX") or "" + if self.conda_env: + self.conda_env = os.path.realpath(self.conda_env) + for conda_path in glob(f"{self.conda_env}/lib/python*/site-packages"): + if conda_path not in self.paths: + self.paths.append(conda_path) + for nested_conda_path in glob(f"{self.conda_env}/lib/python*/*/site-packages"): + if nested_conda_path not in self.paths: + self.paths.append(nested_conda_path) + + # handle case-insensitive paths on windows + self.stdlib_lib_prefix = os.path.normcase(sysconfig.get_paths()["stdlib"]) + if self.stdlib_lib_prefix not in self.paths: + self.paths.append(self.stdlib_lib_prefix) + + # add system paths + for system_path in sys.path[1:]: + if system_path not in self.paths: + self.paths.append(system_path) + + def find(self, module_name: str) -> Optional[str]: + for prefix in self.paths: + package_path = "/".join((prefix, module_name.split(".")[0])) + path_obj = Path(package_path).resolve() + is_module = ( + exists_case_sensitive(package_path + ".py") + or any( + exists_case_sensitive(package_path + ext_suffix) + for ext_suffix in importlib.machinery.EXTENSION_SUFFIXES + ) + or exists_case_sensitive(package_path + "/__init__.py") + ) + is_package = exists_case_sensitive(package_path) and os.path.isdir(package_path) + if is_module or is_package: + if ( + "site-packages" in prefix + or "dist-packages" in prefix + or (self.virtual_env and self.virtual_env_src in prefix) + ): + return sections.THIRDPARTY + elif os.path.normcase(prefix) == self.stdlib_lib_prefix: + return sections.STDLIB + elif self.conda_env and self.conda_env in prefix: + return sections.THIRDPARTY + for src_path in self.config.src_paths: + if src_path in path_obj.parents and not self.config.is_skipped(path_obj): + return sections.FIRSTPARTY + + if os.path.normcase(prefix).startswith(self.stdlib_lib_prefix): + return sections.STDLIB # pragma: no cover - edge case for one OS. Hard to test. + + return self.config.default_section + return None + + +class ReqsBaseFinder(BaseFinder): + enabled = False + + def __init__(self, config: Config, path: str = ".") -> None: + super().__init__(config) + self.path = path + if self.enabled: + self.mapping = self._load_mapping() + self.names = self._load_names() + + @abstractmethod + def _get_names(self, path: str) -> Iterator[str]: + raise NotImplementedError + + @abstractmethod + def _get_files_from_dir(self, path: str) -> Iterator[str]: + raise NotImplementedError + + @staticmethod + def _load_mapping() -> Optional[Dict[str, str]]: + """Return list of mappings `package_name -> module_name` + + Example: + django-haystack -> haystack + """ + if not pipreqs: + return None + path = os.path.dirname(inspect.getfile(pipreqs)) + path = os.path.join(path, "mapping") + with open(path) as f: + mappings: Dict[str, str] = {} # pypi_name: import_name + for line in f: + import_name, _, pypi_name = line.strip().partition(":") + mappings[pypi_name] = import_name + return mappings + # return dict(tuple(line.strip().split(":")[::-1]) for line in f) + + def _load_names(self) -> List[str]: + """Return list of thirdparty modules from requirements""" + names = [] + for path in self._get_files(): + for name in self._get_names(path): + names.append(self._normalize_name(name)) + return names + + @staticmethod + def _get_parents(path: str) -> Iterator[str]: + prev = "" + while path != prev: + prev = path + yield path + path = os.path.dirname(path) + + def _get_files(self) -> Iterator[str]: + """Return paths to all requirements files""" + path = os.path.abspath(self.path) + if os.path.isfile(path): + path = os.path.dirname(path) + + for path in self._get_parents(path): + yield from self._get_files_from_dir(path) + + def _normalize_name(self, name: str) -> str: + """Convert package name to module name + + Examples: + Django -> django + django-haystack -> django_haystack + Flask-RESTFul -> flask_restful + """ + if self.mapping: + name = self.mapping.get(name.replace("-", "_"), name) + return name.lower().replace("-", "_") + + def find(self, module_name: str) -> Optional[str]: + # required lib not installed yet + if not self.enabled: + return None + + module_name, _sep, _submodules = module_name.partition(".") + module_name = module_name.lower() + if not module_name: + return None + + for name in self.names: + if module_name == name: + return sections.THIRDPARTY + return None + + +class RequirementsFinder(ReqsBaseFinder): + exts = (".txt", ".in") + enabled = bool(parse_requirements) + + def _get_files_from_dir(self, path: str) -> Iterator[str]: + """Return paths to requirements files from passed dir.""" + yield from self._get_files_from_dir_cached(path) + + @classmethod + @lru_cache(maxsize=16) + def _get_files_from_dir_cached(cls, path: str) -> List[str]: + results = [] + + for fname in os.listdir(path): + if "requirements" not in fname: + continue + full_path = os.path.join(path, fname) + + # *requirements*/*.{txt,in} + if os.path.isdir(full_path): + for subfile_name in os.listdir(full_path): + for ext in cls.exts: + if subfile_name.endswith(ext): + results.append(os.path.join(full_path, subfile_name)) + continue + + # *requirements*.{txt,in} + if os.path.isfile(full_path): + for ext in cls.exts: + if fname.endswith(ext): + results.append(full_path) + break + + return results + + def _get_names(self, path: str) -> Iterator[str]: + """Load required packages from path to requirements file""" + yield from self._get_names_cached(path) + + @classmethod + @lru_cache(maxsize=16) + def _get_names_cached(cls, path: str) -> List[str]: + result = [] + + with chdir(os.path.dirname(path)): + requirements = parse_requirements(path) + for req in requirements.values(): + if req.name: + result.append(req.name) + + return result + + +class PipfileFinder(ReqsBaseFinder): + enabled = bool(Pipfile) + + def _get_names(self, path: str) -> Iterator[str]: + with chdir(path): + project = Pipfile.load(path) + for req in project.packages: + yield req.name + + def _get_files_from_dir(self, path: str) -> Iterator[str]: + if "Pipfile" in os.listdir(path): + yield path + + +class DefaultFinder(BaseFinder): + def find(self, module_name: str) -> Optional[str]: + return self.config.default_section + + +class FindersManager: + _default_finders_classes: Sequence[Type[BaseFinder]] = ( + ForcedSeparateFinder, + LocalFinder, + KnownPatternFinder, + PathFinder, + PipfileFinder, + RequirementsFinder, + DefaultFinder, + ) + + def __init__( + self, config: Config, finder_classes: Optional[Iterable[Type[BaseFinder]]] = None + ) -> None: + self.verbose: bool = config.verbose + + if finder_classes is None: + finder_classes = self._default_finders_classes + finders: List[BaseFinder] = [] + for finder_cls in finder_classes: + try: + finders.append(finder_cls(config)) + except Exception as exception: + # if one finder fails to instantiate isort can continue using the rest + if self.verbose: + print( + ( + f"{finder_cls.__name__} encountered an error ({exception}) during " + "instantiation and cannot be used" + ) + ) + self.finders: Tuple[BaseFinder, ...] = tuple(finders) + + def find(self, module_name: str) -> Optional[str]: + for finder in self.finders: + try: + section = finder.find(module_name) + if section is not None: + return section + except Exception as exception: + # isort has to be able to keep trying to identify the correct + # import section even if one approach fails + if self.verbose: + print( + f"{finder.__class__.__name__} encountered an error ({exception}) while " + f"trying to identify the {module_name} module" + ) + return None diff --git a/WebCrawler/venv/Lib/site-packages/isort/exceptions.py b/WebCrawler/venv/Lib/site-packages/isort/exceptions.py new file mode 100644 index 0000000..b98454a --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/isort/exceptions.py @@ -0,0 +1,171 @@ +"""All isort specific exception classes should be defined here""" +from pathlib import Path +from typing import Any, Dict, Union + +from .profiles import profiles + + +class ISortError(Exception): + """Base isort exception object from which all isort sourced exceptions should inherit""" + + +class InvalidSettingsPath(ISortError): + """Raised when a settings path is provided that is neither a valid file or directory""" + + def __init__(self, settings_path: str): + super().__init__( + f"isort was told to use the settings_path: {settings_path} as the base directory or " + "file that represents the starting point of config file discovery, but it does not " + "exist." + ) + self.settings_path = settings_path + + +class ExistingSyntaxErrors(ISortError): + """Raised when isort is told to sort imports within code that has existing syntax errors""" + + def __init__(self, file_path: str): + super().__init__( + f"isort was told to sort imports within code that contains syntax errors: " + f"{file_path}." + ) + self.file_path = file_path + + +class IntroducedSyntaxErrors(ISortError): + """Raised when isort has introduced a syntax error in the process of sorting imports""" + + def __init__(self, file_path: str): + super().__init__( + f"isort introduced syntax errors when attempting to sort the imports contained within " + f"{file_path}." + ) + self.file_path = file_path + + +class FileSkipped(ISortError): + """Should be raised when a file is skipped for any reason""" + + def __init__(self, message: str, file_path: str): + super().__init__(message) + self.file_path = file_path + + +class FileSkipComment(FileSkipped): + """Raised when an entire file is skipped due to a isort skip file comment""" + + def __init__(self, file_path: str): + super().__init__( + f"{file_path} contains an file skip comment and was skipped.", file_path=file_path + ) + + +class FileSkipSetting(FileSkipped): + """Raised when an entire file is skipped due to provided isort settings""" + + def __init__(self, file_path: str): + super().__init__( + f"{file_path} was skipped as it's listed in 'skip' setting" + " or matches a glob in 'skip_glob' setting", + file_path=file_path, + ) + + +class ProfileDoesNotExist(ISortError): + """Raised when a profile is set by the user that doesn't exist""" + + def __init__(self, profile: str): + super().__init__( + f"Specified profile of {profile} does not exist. " + f"Available profiles: {','.join(profiles)}." + ) + self.profile = profile + + +class FormattingPluginDoesNotExist(ISortError): + """Raised when a formatting plugin is set by the user that doesn't exist""" + + def __init__(self, formatter: str): + super().__init__(f"Specified formatting plugin of {formatter} does not exist. ") + self.formatter = formatter + + +class LiteralParsingFailure(ISortError): + """Raised when one of isorts literal sorting comments is used but isort can't parse the + the given data structure. + """ + + def __init__(self, code: str, original_error: Exception): + super().__init__( + f"isort failed to parse the given literal {code}. It's important to note " + "that isort literal sorting only supports simple literals parsable by " + f"ast.literal_eval which gave the exception of {original_error}." + ) + self.code = code + self.original_error = original_error + + +class LiteralSortTypeMismatch(ISortError): + """Raised when an isort literal sorting comment is used, with a type that doesn't match the + supplied data structure's type. + """ + + def __init__(self, kind: type, expected_kind: type): + super().__init__( + f"isort was told to sort a literal of type {expected_kind} but was given " + f"a literal of type {kind}." + ) + self.kind = kind + self.expected_kind = expected_kind + + +class AssignmentsFormatMismatch(ISortError): + """Raised when isort is told to sort assignments but the format of the assignment section + doesn't match isort's expectation. + """ + + def __init__(self, code: str): + super().__init__( + "isort was told to sort a section of assignments, however the given code:\n\n" + f"{code}\n\n" + "Does not match isort's strict single line formatting requirement for assignment " + "sorting:\n\n" + "{variable_name} = {value}\n" + "{variable_name2} = {value2}\n" + "...\n\n" + ) + self.code = code + + +class UnsupportedSettings(ISortError): + """Raised when settings are passed into isort (either from config, CLI, or runtime) + that it doesn't support. + """ + + @staticmethod + def _format_option(name: str, value: Any, source: str) -> str: + return f"\t- {name} = {value} (source: '{source}')" + + def __init__(self, unsupported_settings: Dict[str, Dict[str, str]]): + errors = "\n".join( + self._format_option(name, **option) for name, option in unsupported_settings.items() + ) + + super().__init__( + "isort was provided settings that it doesn't support:\n\n" + f"{errors}\n\n" + "For a complete and up-to-date listing of supported settings see: " + "https://pycqa.github.io/isort/docs/configuration/options/.\n" + ) + self.unsupported_settings = unsupported_settings + + +class UnsupportedEncoding(ISortError): + """Raised when isort encounters an encoding error while trying to read a file""" + + def __init__( + self, + filename: Union[str, Path], + ): + super().__init__(f"Unknown or unsupported encoding in {filename}") + self.filename = filename diff --git a/WebCrawler/venv/Lib/site-packages/isort/format.py b/WebCrawler/venv/Lib/site-packages/isort/format.py new file mode 100644 index 0000000..46bb156 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/isort/format.py @@ -0,0 +1,149 @@ +import re +import sys +from datetime import datetime +from difflib import unified_diff +from pathlib import Path +from typing import Optional, TextIO + +try: + import colorama +except ImportError: + colorama_unavailable = True +else: + colorama_unavailable = False + colorama.init() + + +ADDED_LINE_PATTERN = re.compile(r"\+[^+]") +REMOVED_LINE_PATTERN = re.compile(r"-[^-]") + + +def format_simplified(import_line: str) -> str: + import_line = import_line.strip() + if import_line.startswith("from "): + import_line = import_line.replace("from ", "") + import_line = import_line.replace(" import ", ".") + elif import_line.startswith("import "): + import_line = import_line.replace("import ", "") + + return import_line + + +def format_natural(import_line: str) -> str: + import_line = import_line.strip() + if not import_line.startswith("from ") and not import_line.startswith("import "): + if "." not in import_line: + return f"import {import_line}" + parts = import_line.split(".") + end = parts.pop(-1) + return f"from {'.'.join(parts)} import {end}" + + return import_line + + +def show_unified_diff( + *, + file_input: str, + file_output: str, + file_path: Optional[Path], + output: Optional[TextIO] = None, + color_output: bool = False, +): + """Shows a unified_diff for the provided input and output against the provided file path. + + - **file_input**: A string that represents the contents of a file before changes. + - **file_output**: A string that represents the contents of a file after changes. + - **file_path**: A Path object that represents the file path of the file being changed. + - **output**: A stream to output the diff to. If non is provided uses sys.stdout. + - **color_output**: Use color in output if True. + """ + printer = create_terminal_printer(color_output, output) + file_name = "" if file_path is None else str(file_path) + file_mtime = str( + datetime.now() if file_path is None else datetime.fromtimestamp(file_path.stat().st_mtime) + ) + unified_diff_lines = unified_diff( + file_input.splitlines(keepends=True), + file_output.splitlines(keepends=True), + fromfile=file_name + ":before", + tofile=file_name + ":after", + fromfiledate=file_mtime, + tofiledate=str(datetime.now()), + ) + for line in unified_diff_lines: + printer.diff_line(line) + + +def ask_whether_to_apply_changes_to_file(file_path: str) -> bool: + answer = None + while answer not in ("yes", "y", "no", "n", "quit", "q"): + answer = input(f"Apply suggested changes to '{file_path}' [y/n/q]? ") # nosec + answer = answer.lower() + if answer in ("no", "n"): + return False + if answer in ("quit", "q"): + sys.exit(1) + return True + + +def remove_whitespace(content: str, line_separator: str = "\n") -> str: + content = content.replace(line_separator, "").replace(" ", "").replace("\x0c", "") + return content + + +class BasicPrinter: + ERROR = "ERROR" + SUCCESS = "SUCCESS" + + def __init__(self, output: Optional[TextIO] = None): + self.output = output or sys.stdout + + def success(self, message: str) -> None: + print(f"{self.SUCCESS}: {message}", file=self.output) + + def error(self, message: str) -> None: + print(f"{self.ERROR}: {message}", file=sys.stderr) + + def diff_line(self, line: str) -> None: + self.output.write(line) + + +class ColoramaPrinter(BasicPrinter): + def __init__(self, output: Optional[TextIO] = None): + self.output = output or sys.stdout + # Note: this constants are instance variables instead ofs class variables + # because they refer to colorama which might not be installed. + self.ERROR = self.style_text("ERROR", colorama.Fore.RED) + self.SUCCESS = self.style_text("SUCCESS", colorama.Fore.GREEN) + self.ADDED_LINE = colorama.Fore.GREEN + self.REMOVED_LINE = colorama.Fore.RED + + @staticmethod + def style_text(text: str, style: Optional[str] = None) -> str: + if style is None: + return text + return style + text + colorama.Style.RESET_ALL + + def diff_line(self, line: str) -> None: + style = None + if re.match(ADDED_LINE_PATTERN, line): + style = self.ADDED_LINE + elif re.match(REMOVED_LINE_PATTERN, line): + style = self.REMOVED_LINE + self.output.write(self.style_text(line, style)) + + +def create_terminal_printer(color: bool, output: Optional[TextIO] = None): + if color and colorama_unavailable: + no_colorama_message = ( + "\n" + "Sorry, but to use --color (color_output) the colorama python package is required.\n\n" + "Reference: https://pypi.org/project/colorama/\n\n" + "You can either install it separately on your system or as the colors extra " + "for isort. Ex: \n\n" + "$ pip install isort[colors]\n" + ) + print(no_colorama_message, file=sys.stderr) + sys.exit(1) + + return ColoramaPrinter(output) if color else BasicPrinter(output) diff --git a/WebCrawler/venv/Lib/site-packages/isort/hooks.py b/WebCrawler/venv/Lib/site-packages/isort/hooks.py new file mode 100644 index 0000000..acccede --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/isort/hooks.py @@ -0,0 +1,90 @@ +"""Defines a git hook to allow pre-commit warnings and errors about import order. + +usage: + exit_code = git_hook(strict=True|False, modify=True|False) +""" +import os +import subprocess # nosec - Needed for hook +from pathlib import Path +from typing import List + +from isort import Config, api, exceptions + + +def get_output(command: List[str]) -> str: + """ + Run a command and return raw output + + :param str command: the command to run + :returns: the stdout output of the command + """ + result = subprocess.run(command, stdout=subprocess.PIPE, check=True) # nosec - trusted input + return result.stdout.decode() + + +def get_lines(command: List[str]) -> List[str]: + """ + Run a command and return lines of output + + :param str command: the command to run + :returns: list of whitespace-stripped lines output by command + """ + stdout = get_output(command) + return [line.strip() for line in stdout.splitlines()] + + +def git_hook( + strict: bool = False, modify: bool = False, lazy: bool = False, settings_file: str = "" +) -> int: + """ + Git pre-commit hook to check staged files for isort errors + + :param bool strict - if True, return number of errors on exit, + causing the hook to fail. If False, return zero so it will + just act as a warning. + :param bool modify - if True, fix the sources if they are not + sorted properly. If False, only report result without + modifying anything. + :param bool lazy - if True, also check/fix unstaged files. + This is useful if you frequently use ``git commit -a`` for example. + If False, ony check/fix the staged files for isort errors. + :param str settings_file - A path to a file to be used as + the configuration file for this run. + When settings_file is the empty string, the configuration file + will be searched starting at the directory containing the first + staged file, if any, and going upward in the directory structure. + + :return number of errors if in strict mode, 0 otherwise. + """ + + # Get list of files modified and staged + diff_cmd = ["git", "diff-index", "--cached", "--name-only", "--diff-filter=ACMRTUXB", "HEAD"] + if lazy: + diff_cmd.remove("--cached") + + files_modified = get_lines(diff_cmd) + if not files_modified: + return 0 + + errors = 0 + config = Config( + settings_file=settings_file, + settings_path=os.path.dirname(os.path.abspath(files_modified[0])), + ) + for filename in files_modified: + if filename.endswith(".py"): + # Get the staged contents of the file + staged_cmd = ["git", "show", f":{filename}"] + staged_contents = get_output(staged_cmd) + + try: + if not api.check_code_string( + staged_contents, file_path=Path(filename), config=config + ): + errors += 1 + if modify: + api.sort_file(filename, config=config) + except exceptions.FileSkipped: # pragma: no cover + pass + + return errors if strict else 0 diff --git a/WebCrawler/venv/Lib/site-packages/isort/io.py b/WebCrawler/venv/Lib/site-packages/isort/io.py new file mode 100644 index 0000000..7ff2807 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/isort/io.py @@ -0,0 +1,69 @@ +"""Defines any IO utilities used by isort""" +import re +import tokenize +from contextlib import contextmanager +from io import BytesIO, StringIO, TextIOWrapper +from pathlib import Path +from typing import Callable, Iterator, NamedTuple, TextIO, Union + +from isort.exceptions import UnsupportedEncoding + +_ENCODING_PATTERN = re.compile(br"^[ \t\f]*#.*?coding[:=][ \t]*([-_.a-zA-Z0-9]+)") + + +class File(NamedTuple): + stream: TextIO + path: Path + encoding: str + + @staticmethod + def detect_encoding(filename: str, readline: Callable[[], bytes]): + try: + return tokenize.detect_encoding(readline)[0] + except Exception: + raise UnsupportedEncoding(filename) + + @staticmethod + def from_contents(contents: str, filename: str) -> "File": + encoding = File.detect_encoding(filename, BytesIO(contents.encode("utf-8")).readline) + return File(StringIO(contents), path=Path(filename).resolve(), encoding=encoding) + + @property + def extension(self): + return self.path.suffix.lstrip(".") + + @staticmethod + def _open(filename): + """Open a file in read only mode using the encoding detected by + detect_encoding(). + """ + buffer = open(filename, "rb") + try: + encoding = File.detect_encoding(filename, buffer.readline) + buffer.seek(0) + text = TextIOWrapper(buffer, encoding, line_buffering=True, newline="") + text.mode = "r" # type: ignore + return text + except Exception: + buffer.close() + raise + + @staticmethod + @contextmanager + def read(filename: Union[str, Path]) -> Iterator["File"]: + file_path = Path(filename).resolve() + stream = None + try: + stream = File._open(file_path) + yield File(stream=stream, path=file_path, encoding=stream.encoding) + finally: + if stream is not None: + stream.close() + + +class _EmptyIO(StringIO): + def write(self, *args, **kwargs): + pass + + +Empty = _EmptyIO() diff --git a/WebCrawler/venv/Lib/site-packages/isort/literal.py b/WebCrawler/venv/Lib/site-packages/isort/literal.py new file mode 100644 index 0000000..01bd05e --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/isort/literal.py @@ -0,0 +1,109 @@ +import ast +from pprint import PrettyPrinter +from typing import Any, Callable, Dict, List, Set, Tuple + +from isort.exceptions import ( + AssignmentsFormatMismatch, + LiteralParsingFailure, + LiteralSortTypeMismatch, +) +from isort.settings import DEFAULT_CONFIG, Config + + +class ISortPrettyPrinter(PrettyPrinter): + """an isort customized pretty printer for sorted literals""" + + def __init__(self, config: Config): + super().__init__(width=config.line_length, compact=True) + + +type_mapping: Dict[str, Tuple[type, Callable[[Any, ISortPrettyPrinter], str]]] = {} + + +def assignments(code: str) -> str: + values = {} + for line in code.splitlines(keepends=True): + if not line.strip(): + continue + if " = " not in line: + raise AssignmentsFormatMismatch(code) + variable_name, value = line.split(" = ", 1) + values[variable_name] = value + + return "".join( + f"{variable_name} = {values[variable_name]}" for variable_name in sorted(values.keys()) + ) + + +def assignment(code: str, sort_type: str, extension: str, config: Config = DEFAULT_CONFIG) -> str: + """Sorts the literal present within the provided code against the provided sort type, + returning the sorted representation of the source code. + """ + if sort_type == "assignments": + return assignments(code) + elif sort_type not in type_mapping: + raise ValueError( + "Trying to sort using an undefined sort_type. " + f"Defined sort types are {', '.join(type_mapping.keys())}." + ) + + variable_name, literal = code.split(" = ") + variable_name = variable_name.lstrip() + try: + value = ast.literal_eval(literal) + except Exception as error: + raise LiteralParsingFailure(code, error) + + expected_type, sort_function = type_mapping[sort_type] + if type(value) != expected_type: + raise LiteralSortTypeMismatch(type(value), expected_type) + + printer = ISortPrettyPrinter(config) + sorted_value_code = f"{variable_name} = {sort_function(value, printer)}" + if config.formatting_function: + sorted_value_code = config.formatting_function( + sorted_value_code, extension, config + ).rstrip() + + sorted_value_code += code[len(code.rstrip()) :] + return sorted_value_code + + +def register_type(name: str, kind: type): + """Registers a new literal sort type.""" + + def wrap(function): + type_mapping[name] = (kind, function) + return function + + return wrap + + +@register_type("dict", dict) +def _dict(value: Dict[Any, Any], printer: ISortPrettyPrinter) -> str: + return printer.pformat(dict(sorted(value.items(), key=lambda item: item[1]))) + + +@register_type("list", list) +def _list(value: List[Any], printer: ISortPrettyPrinter) -> str: + return printer.pformat(sorted(value)) + + +@register_type("unique-list", list) +def _unique_list(value: List[Any], printer: ISortPrettyPrinter) -> str: + return printer.pformat(list(sorted(set(value)))) + + +@register_type("set", set) +def _set(value: Set[Any], printer: ISortPrettyPrinter) -> str: + return "{" + printer.pformat(tuple(sorted(value)))[1:-1] + "}" + + +@register_type("tuple", tuple) +def _tuple(value: Tuple[Any, ...], printer: ISortPrettyPrinter) -> str: + return printer.pformat(tuple(sorted(value))) + + +@register_type("unique-tuple", tuple) +def _unique_tuple(value: Tuple[Any, ...], printer: ISortPrettyPrinter) -> str: + return printer.pformat(tuple(sorted(set(value)))) diff --git a/WebCrawler/venv/Lib/site-packages/isort/logo.py b/WebCrawler/venv/Lib/site-packages/isort/logo.py new file mode 100644 index 0000000..6377d86 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/isort/logo.py @@ -0,0 +1,19 @@ +from ._version import __version__ + +ASCII_ART = rf""" + _ _ + (_) ___ ___ _ __| |_ + | |/ _/ / _ \/ '__ _/ + | |\__ \/\_\/| | | |_ + |_|\___/\___/\_/ \_/ + + isort your imports, so you don't have to. + + VERSION {__version__} +""" + +__doc__ = f""" +```python +{ASCII_ART} +``` +""" diff --git a/WebCrawler/venv/Lib/site-packages/isort/main.py b/WebCrawler/venv/Lib/site-packages/isort/main.py new file mode 100644 index 0000000..a8e59f0 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/isort/main.py @@ -0,0 +1,1022 @@ +"""Tool for sorting imports alphabetically, and automatically separated into sections.""" +import argparse +import functools +import json +import os +import sys +from io import TextIOWrapper +from pathlib import Path +from typing import Any, Dict, Iterable, Iterator, List, Optional, Sequence, Set +from warnings import warn + +from . import __version__, api, sections +from .exceptions import FileSkipped, UnsupportedEncoding +from .format import create_terminal_printer +from .logo import ASCII_ART +from .profiles import profiles +from .settings import VALID_PY_TARGETS, Config, WrapModes + +try: + from .setuptools_commands import ISortCommand # noqa: F401 +except ImportError: + pass + +DEPRECATED_SINGLE_DASH_ARGS = { + "-ac", + "-af", + "-ca", + "-cs", + "-df", + "-ds", + "-dt", + "-fas", + "-fass", + "-ff", + "-fgw", + "-fss", + "-lai", + "-lbt", + "-le", + "-ls", + "-nis", + "-nlb", + "-ot", + "-rr", + "-sd", + "-sg", + "-sl", + "-sp", + "-tc", + "-wl", + "-ws", +} +QUICK_GUIDE = f""" +{ASCII_ART} + +Nothing to do: no files or paths have have been passed in! + +Try one of the following: + + `isort .` - sort all Python files, starting from the current directory, recursively. + `isort . --interactive` - Do the same, but ask before making any changes. + `isort . --check --diff` - Check to see if imports are correctly sorted within this project. + `isort --help` - In-depth information about isort's available command-line options. + +Visit https://pycqa.github.io/isort/ for complete information about how to use isort. +""" + + +class SortAttempt: + def __init__(self, incorrectly_sorted: bool, skipped: bool, supported_encoding: bool) -> None: + self.incorrectly_sorted = incorrectly_sorted + self.skipped = skipped + self.supported_encoding = supported_encoding + + +def sort_imports( + file_name: str, + config: Config, + check: bool = False, + ask_to_apply: bool = False, + write_to_stdout: bool = False, + **kwargs: Any, +) -> Optional[SortAttempt]: + try: + incorrectly_sorted: bool = False + skipped: bool = False + if check: + try: + incorrectly_sorted = not api.check_file(file_name, config=config, **kwargs) + except FileSkipped: + skipped = True + return SortAttempt(incorrectly_sorted, skipped, True) + else: + try: + incorrectly_sorted = not api.sort_file( + file_name, + config=config, + ask_to_apply=ask_to_apply, + write_to_stdout=write_to_stdout, + **kwargs, + ) + except FileSkipped: + skipped = True + return SortAttempt(incorrectly_sorted, skipped, True) + except (OSError, ValueError) as error: + warn(f"Unable to parse file {file_name} due to {error}") + return None + except UnsupportedEncoding: + if config.verbose: + warn(f"Encoding not supported for {file_name}") + return SortAttempt(incorrectly_sorted, skipped, False) + except Exception: + printer = create_terminal_printer(color=config.color_output) + printer.error( + f"Unrecoverable exception thrown when parsing {file_name}! " + "This should NEVER happen.\n" + "If encountered, please open an issue: https://github.com/PyCQA/isort/issues/new" + ) + raise + + +def iter_source_code( + paths: Iterable[str], config: Config, skipped: List[str], broken: List[str] +) -> Iterator[str]: + """Iterate over all Python source files defined in paths.""" + visited_dirs: Set[Path] = set() + + for path in paths: + if os.path.isdir(path): + for dirpath, dirnames, filenames in os.walk(path, topdown=True, followlinks=True): + base_path = Path(dirpath) + for dirname in list(dirnames): + full_path = base_path / dirname + resolved_path = full_path.resolve() + if config.is_skipped(full_path): + skipped.append(dirname) + dirnames.remove(dirname) + else: + if resolved_path in visited_dirs: # pragma: no cover + if not config.quiet: + warn(f"Likely recursive symlink detected to {resolved_path}") + dirnames.remove(dirname) + visited_dirs.add(resolved_path) + + for filename in filenames: + filepath = os.path.join(dirpath, filename) + if config.is_supported_filetype(filepath): + if config.is_skipped(Path(filepath)): + skipped.append(filename) + else: + yield filepath + elif not os.path.exists(path): + broken.append(path) + else: + yield path + + +def _build_arg_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description="Sort Python import definitions alphabetically " + "within logical sections. Run with no arguments to see a quick " + "start guide, otherwise, one or more files/directories/stdin must be provided. " + "Use `-` as the first argument to represent stdin. Use --interactive to use the pre 5.0.0 " + "interactive behavior." + " " + "If you've used isort 4 but are new to isort 5, see the upgrading guide:" + "https://pycqa.github.io/isort/docs/upgrade_guides/5.0.0/." + ) + inline_args_group = parser.add_mutually_exclusive_group() + parser.add_argument( + "--src", + "--src-path", + dest="src_paths", + action="append", + help="Add an explicitly defined source path " + "(modules within src paths have their imports automatically categorized as first_party).", + ) + parser.add_argument( + "-a", + "--add-import", + dest="add_imports", + action="append", + help="Adds the specified import line to all files, " + "automatically determining correct placement.", + ) + parser.add_argument( + "--append", + "--append-only", + dest="append_only", + action="store_true", + help="Only adds the imports specified in --add-imports if the file" + " contains existing imports.", + ) + parser.add_argument( + "--ac", + "--atomic", + dest="atomic", + action="store_true", + help="Ensures the output doesn't save if the resulting file contains syntax errors.", + ) + parser.add_argument( + "--af", + "--force-adds", + dest="force_adds", + action="store_true", + help="Forces import adds even if the original file is empty.", + ) + parser.add_argument( + "-b", + "--builtin", + dest="known_standard_library", + action="append", + help="Force isort to recognize a module as part of Python's standard library.", + ) + parser.add_argument( + "--extra-builtin", + dest="extra_standard_library", + action="append", + help="Extra modules to be included in the list of ones in Python's standard library.", + ) + parser.add_argument( + "-c", + "--check-only", + "--check", + action="store_true", + dest="check", + help="Checks the file for unsorted / unformatted imports and prints them to the " + "command line without modifying the file.", + ) + parser.add_argument( + "--ca", + "--combine-as", + dest="combine_as_imports", + action="store_true", + help="Combines as imports on the same line.", + ) + parser.add_argument( + "--cs", + "--combine-star", + dest="combine_star", + action="store_true", + help="Ensures that if a star import is present, " + "nothing else is imported from that namespace.", + ) + parser.add_argument( + "-d", + "--stdout", + help="Force resulting output to stdout, instead of in-place.", + dest="write_to_stdout", + action="store_true", + ) + parser.add_argument( + "--df", + "--diff", + dest="show_diff", + action="store_true", + help="Prints a diff of all the changes isort would make to a file, instead of " + "changing it in place", + ) + parser.add_argument( + "--ds", + "--no-sections", + help="Put all imports into the same section bucket", + dest="no_sections", + action="store_true", + ) + parser.add_argument( + "-e", + "--balanced", + dest="balanced_wrapping", + action="store_true", + help="Balances wrapping to produce the most consistent line length possible", + ) + parser.add_argument( + "-f", + "--future", + dest="known_future_library", + action="append", + help="Force isort to recognize a module as part of Python's internal future compatibility " + "libraries. WARNING: this overrides the behavior of __future__ handling and therefore" + " can result in code that can't execute. If you're looking to add dependencies such " + "as six a better option is to create a another section below --future using custom " + "sections. See: https://github.com/PyCQA/isort#custom-sections-and-ordering and the " + "discussion here: https://github.com/PyCQA/isort/issues/1463.", + ) + parser.add_argument( + "--fas", + "--force-alphabetical-sort", + action="store_true", + dest="force_alphabetical_sort", + help="Force all imports to be sorted as a single section", + ) + parser.add_argument( + "--fass", + "--force-alphabetical-sort-within-sections", + action="store_true", + dest="force_alphabetical_sort_within_sections", + help="Force all imports to be sorted alphabetically within a section", + ) + parser.add_argument( + "--ff", + "--from-first", + dest="from_first", + help="Switches the typical ordering preference, " + "showing from imports first then straight ones.", + ) + parser.add_argument( + "--fgw", + "--force-grid-wrap", + nargs="?", + const=2, + type=int, + dest="force_grid_wrap", + help="Force number of from imports (defaults to 2 when passed as CLI flag without value)" + "to be grid wrapped regardless of line " + "length. If 0 is passed in (the global default) only line length is considered.", + ) + parser.add_argument( + "--fss", + "--force-sort-within-sections", + action="store_true", + dest="force_sort_within_sections", + help="Don't sort straight-style imports (like import sys) before from-style imports " + "(like from itertools import groupby). Instead, sort the imports by module, " + "independent of import style.", + ) + parser.add_argument( + "-i", + "--indent", + help='String to place for indents defaults to " " (4 spaces).', + dest="indent", + type=str, + ) + parser.add_argument( + "-j", "--jobs", help="Number of files to process in parallel.", dest="jobs", type=int + ) + parser.add_argument("--lai", "--lines-after-imports", dest="lines_after_imports", type=int) + parser.add_argument("--lbt", "--lines-between-types", dest="lines_between_types", type=int) + parser.add_argument( + "--le", + "--line-ending", + dest="line_ending", + help="Forces line endings to the specified value. " + "If not set, values will be guessed per-file.", + ) + parser.add_argument( + "--ls", + "--length-sort", + help="Sort imports by their string length.", + dest="length_sort", + action="store_true", + ) + parser.add_argument( + "--lss", + "--length-sort-straight", + help="Sort straight imports by their string length. Similar to `length_sort` " + "but applies only to straight imports and doesn't affect from imports.", + dest="length_sort_straight", + action="store_true", + ) + parser.add_argument( + "-m", + "--multi-line", + dest="multi_line_output", + choices=list(WrapModes.__members__.keys()) + + [str(mode.value) for mode in WrapModes.__members__.values()], + type=str, + help="Multi line output (0-grid, 1-vertical, 2-hanging, 3-vert-hanging, 4-vert-grid, " + "5-vert-grid-grouped, 6-vert-grid-grouped-no-comma, 7-noqa, " + "8-vertical-hanging-indent-bracket, 9-vertical-prefix-from-module-import, " + "10-hanging-indent-with-parentheses).", + ) + parser.add_argument( + "-n", + "--ensure-newline-before-comments", + dest="ensure_newline_before_comments", + action="store_true", + help="Inserts a blank line before a comment following an import.", + ) + inline_args_group.add_argument( + "--nis", + "--no-inline-sort", + dest="no_inline_sort", + action="store_true", + help="Leaves `from` imports with multiple imports 'as-is' " + "(e.g. `from foo import a, c ,b`).", + ) + parser.add_argument( + "--nlb", + "--no-lines-before", + help="Sections which should not be split with previous by empty lines", + dest="no_lines_before", + action="append", + ) + parser.add_argument( + "-o", + "--thirdparty", + dest="known_third_party", + action="append", + help="Force isort to recognize a module as being part of a third party library.", + ) + parser.add_argument( + "--ot", + "--order-by-type", + dest="order_by_type", + action="store_true", + help="Order imports by type, which is determined by case, in addition to alphabetically.\n" + "\n**NOTE**: type here refers to the implied type from the import name capitalization.\n" + ' isort does not do type introspection for the imports. These "types" are simply: ' + "CONSTANT_VARIABLE, CamelCaseClass, variable_or_function. If your project follows PEP8" + " or a related coding standard and has many imports this is a good default, otherwise you " + "likely will want to turn it off. From the CLI the `--dont-order-by-type` option will turn " + "this off.", + ) + parser.add_argument( + "--dt", + "--dont-order-by-type", + dest="dont_order_by_type", + action="store_true", + help="Don't order imports by type, which is determined by case, in addition to " + "alphabetically.\n\n" + "**NOTE**: type here refers to the implied type from the import name capitalization.\n" + ' isort does not do type introspection for the imports. These "types" are simply: ' + "CONSTANT_VARIABLE, CamelCaseClass, variable_or_function. If your project follows PEP8" + " or a related coding standard and has many imports this is a good default. You can turn " + "this on from the CLI using `--order-by-type`.", + ) + parser.add_argument( + "-p", + "--project", + dest="known_first_party", + action="append", + help="Force isort to recognize a module as being part of the current python project.", + ) + parser.add_argument( + "--known-local-folder", + dest="known_local_folder", + action="append", + help="Force isort to recognize a module as being a local folder. " + "Generally, this is reserved for relative imports (from . import module).", + ) + parser.add_argument( + "-q", + "--quiet", + action="store_true", + dest="quiet", + help="Shows extra quiet output, only errors are outputted.", + ) + parser.add_argument( + "--rm", + "--remove-import", + dest="remove_imports", + action="append", + help="Removes the specified import from all files.", + ) + parser.add_argument( + "--rr", + "--reverse-relative", + dest="reverse_relative", + action="store_true", + help="Reverse order of relative imports.", + ) + parser.add_argument( + "-s", + "--skip", + help="Files that sort imports should skip over. If you want to skip multiple " + "files you should specify twice: --skip file1 --skip file2.", + dest="skip", + action="append", + ) + parser.add_argument( + "--sd", + "--section-default", + dest="default_section", + help="Sets the default section for import options: " + str(sections.DEFAULT), + ) + parser.add_argument( + "--sg", + "--skip-glob", + help="Files that sort imports should skip over.", + dest="skip_glob", + action="append", + ) + parser.add_argument( + "--gitignore", + "--skip-gitignore", + action="store_true", + dest="skip_gitignore", + help="Treat project as a git repository and ignore files listed in .gitignore", + ) + inline_args_group.add_argument( + "--sl", + "--force-single-line-imports", + dest="force_single_line", + action="store_true", + help="Forces all from imports to appear on their own line", + ) + parser.add_argument( + "--nsl", + "--single-line-exclusions", + help="One or more modules to exclude from the single line rule.", + dest="single_line_exclusions", + action="append", + ) + parser.add_argument( + "--sp", + "--settings-path", + "--settings-file", + "--settings", + dest="settings_path", + help="Explicitly set the settings path or file instead of auto determining " + "based on file location.", + ) + parser.add_argument( + "-t", + "--top", + help="Force specific imports to the top of their appropriate section.", + dest="force_to_top", + action="append", + ) + parser.add_argument( + "--tc", + "--trailing-comma", + dest="include_trailing_comma", + action="store_true", + help="Includes a trailing comma on multi line imports that include parentheses.", + ) + parser.add_argument( + "--up", + "--use-parentheses", + dest="use_parentheses", + action="store_true", + help="Use parentheses for line continuation on length limit instead of slashes." + " **NOTE**: This is separate from wrap modes, and only affects how individual lines that " + " are too long get continued, not sections of multiple imports.", + ) + parser.add_argument( + "-V", + "--version", + action="store_true", + dest="show_version", + help="Displays the currently installed version of isort.", + ) + parser.add_argument( + "-v", + "--verbose", + action="store_true", + dest="verbose", + help="Shows verbose output, such as when files are skipped or when a check is successful.", + ) + parser.add_argument( + "--virtual-env", + dest="virtual_env", + help="Virtual environment to use for determining whether a package is third-party", + ) + parser.add_argument( + "--conda-env", + dest="conda_env", + help="Conda environment to use for determining whether a package is third-party", + ) + parser.add_argument( + "--vn", + "--version-number", + action="version", + version=__version__, + help="Returns just the current version number without the logo", + ) + parser.add_argument( + "-l", + "-w", + "--line-length", + "--line-width", + help="The max length of an import line (used for wrapping long imports).", + dest="line_length", + type=int, + ) + parser.add_argument( + "--wl", + "--wrap-length", + dest="wrap_length", + type=int, + help="Specifies how long lines that are wrapped should be, if not set line_length is used." + "\nNOTE: wrap_length must be LOWER than or equal to line_length.", + ) + parser.add_argument( + "--ws", + "--ignore-whitespace", + action="store_true", + dest="ignore_whitespace", + help="Tells isort to ignore whitespace differences when --check-only is being used.", + ) + parser.add_argument( + "--case-sensitive", + dest="case_sensitive", + action="store_true", + help="Tells isort to include casing when sorting module names", + ) + parser.add_argument( + "--filter-files", + dest="filter_files", + action="store_true", + help="Tells isort to filter files even when they are explicitly passed in as " + "part of the CLI command.", + ) + parser.add_argument( + "files", nargs="*", help="One or more Python source files that need their imports sorted." + ) + parser.add_argument( + "--py", + "--python-version", + action="store", + dest="py_version", + choices=tuple(VALID_PY_TARGETS) + ("auto",), + help="Tells isort to set the known standard library based on the specified Python " + "version. Default is to assume any Python 3 version could be the target, and use a union " + "of all stdlib modules across versions. If auto is specified, the version of the " + "interpreter used to run isort " + f"(currently: {sys.version_info.major}{sys.version_info.minor}) will be used.", + ) + parser.add_argument( + "--profile", + dest="profile", + type=str, + help="Base profile type to use for configuration. " + f"Profiles include: {', '.join(profiles.keys())}. As well as any shared profiles.", + ) + parser.add_argument( + "--interactive", + dest="ask_to_apply", + action="store_true", + help="Tells isort to apply changes interactively.", + ) + parser.add_argument( + "--old-finders", + "--magic-placement", + dest="old_finders", + action="store_true", + help="Use the old deprecated finder logic that relies on environment introspection magic.", + ) + parser.add_argument( + "--show-config", + dest="show_config", + action="store_true", + help="See isort's determined config, as well as sources of config options.", + ) + parser.add_argument( + "--show-files", + dest="show_files", + action="store_true", + help="See the files isort will be ran against with the current config options.", + ) + parser.add_argument( + "--honor-noqa", + dest="honor_noqa", + action="store_true", + help="Tells isort to honor noqa comments to enforce skipping those comments.", + ) + parser.add_argument( + "--remove-redundant-aliases", + dest="remove_redundant_aliases", + action="store_true", + help=( + "Tells isort to remove redundant aliases from imports, such as `import os as os`." + " This defaults to `False` simply because some projects use these seemingly useless " + " aliases to signify intent and change behaviour." + ), + ) + parser.add_argument( + "--color", + dest="color_output", + action="store_true", + help="Tells isort to use color in terminal output.", + ) + parser.add_argument( + "--float-to-top", + dest="float_to_top", + action="store_true", + help="Causes all non-indented imports to float to the top of the file having its imports " + "sorted (immediately below the top of file comment).\n" + "This can be an excellent shortcut for collecting imports every once in a while " + "when you place them in the middle of a file to avoid context switching.\n\n" + "*NOTE*: It currently doesn't work with cimports and introduces some extra over-head " + "and a performance penalty.", + ) + parser.add_argument( + "--treat-comment-as-code", + dest="treat_comments_as_code", + action="append", + help="Tells isort to treat the specified single line comment(s) as if they are code.", + ) + parser.add_argument( + "--treat-all-comment-as-code", + dest="treat_all_comments_as_code", + action="store_true", + help="Tells isort to treat all single line comments as if they are code.", + ) + parser.add_argument( + "--formatter", + dest="formatter", + type=str, + help="Specifies the name of a formatting plugin to use when producing output.", + ) + parser.add_argument( + "--ext", + "--extension", + "--supported-extension", + dest="supported_extensions", + action="append", + help="Specifies what extensions isort can be ran against.", + ) + parser.add_argument( + "--blocked-extension", + dest="blocked_extensions", + action="append", + help="Specifies what extensions isort can never be ran against.", + ) + parser.add_argument( + "--dedup-headings", + dest="dedup_headings", + action="store_true", + help="Tells isort to only show an identical custom import heading comment once, even if" + " there are multiple sections with the comment set.", + ) + + # deprecated options + parser.add_argument( + "--recursive", + dest="deprecated_flags", + action="append_const", + const="--recursive", + help=argparse.SUPPRESS, + ) + parser.add_argument( + "-rc", dest="deprecated_flags", action="append_const", const="-rc", help=argparse.SUPPRESS + ) + parser.add_argument( + "--dont-skip", + dest="deprecated_flags", + action="append_const", + const="--dont-skip", + help=argparse.SUPPRESS, + ) + parser.add_argument( + "-ns", dest="deprecated_flags", action="append_const", const="-ns", help=argparse.SUPPRESS + ) + parser.add_argument( + "--apply", + dest="deprecated_flags", + action="append_const", + const="--apply", + help=argparse.SUPPRESS, + ) + parser.add_argument( + "-k", + "--keep-direct-and-as", + dest="deprecated_flags", + action="append_const", + const="--keep-direct-and-as", + help=argparse.SUPPRESS, + ) + + parser.add_argument( + "--only-sections", + "--os", + dest="only_sections", + action="store_true", + help="Causes imports to be sorted only based on their sections like STDLIB,THIRDPARTY etc. " + "Imports are unaltered and keep their relative positions within the different sections.", + ) + + parser.add_argument( + "--only-modified", + "--om", + dest="only_modified", + action="store_true", + help="Suppresses verbose output for non-modified files.", + ) + + return parser + + +def parse_args(argv: Optional[Sequence[str]] = None) -> Dict[str, Any]: + argv = sys.argv[1:] if argv is None else list(argv) + remapped_deprecated_args = [] + for index, arg in enumerate(argv): + if arg in DEPRECATED_SINGLE_DASH_ARGS: + remapped_deprecated_args.append(arg) + argv[index] = f"-{arg}" + + parser = _build_arg_parser() + arguments = {key: value for key, value in vars(parser.parse_args(argv)).items() if value} + if remapped_deprecated_args: + arguments["remapped_deprecated_args"] = remapped_deprecated_args + if "dont_order_by_type" in arguments: + arguments["order_by_type"] = False + del arguments["dont_order_by_type"] + multi_line_output = arguments.get("multi_line_output", None) + if multi_line_output: + if multi_line_output.isdigit(): + arguments["multi_line_output"] = WrapModes(int(multi_line_output)) + else: + arguments["multi_line_output"] = WrapModes[multi_line_output] + return arguments + + +def _preconvert(item): + """Preconverts objects from native types into JSONifyiable types""" + if isinstance(item, (set, frozenset)): + return list(item) + elif isinstance(item, WrapModes): + return item.name + elif isinstance(item, Path): + return str(item) + elif callable(item) and hasattr(item, "__name__"): + return item.__name__ + else: + raise TypeError("Unserializable object {} of type {}".format(item, type(item))) + + +def main(argv: Optional[Sequence[str]] = None, stdin: Optional[TextIOWrapper] = None) -> None: + arguments = parse_args(argv) + if arguments.get("show_version"): + print(ASCII_ART) + return + + show_config: bool = arguments.pop("show_config", False) + show_files: bool = arguments.pop("show_files", False) + if show_config and show_files: + sys.exit("Error: either specify show-config or show-files not both.") + + if "settings_path" in arguments: + if os.path.isfile(arguments["settings_path"]): + arguments["settings_file"] = os.path.abspath(arguments["settings_path"]) + arguments["settings_path"] = os.path.dirname(arguments["settings_file"]) + else: + arguments["settings_path"] = os.path.abspath(arguments["settings_path"]) + + if "virtual_env" in arguments: + venv = arguments["virtual_env"] + arguments["virtual_env"] = os.path.abspath(venv) + if not os.path.isdir(arguments["virtual_env"]): + warn(f"virtual_env dir does not exist: {arguments['virtual_env']}") + + file_names = arguments.pop("files", []) + if not file_names and not show_config: + print(QUICK_GUIDE) + if arguments: + sys.exit("Error: arguments passed in without any paths or content.") + else: + return + if "settings_path" not in arguments: + arguments["settings_path"] = ( + os.path.abspath(file_names[0] if file_names else ".") or os.getcwd() + ) + if not os.path.isdir(arguments["settings_path"]): + arguments["settings_path"] = os.path.dirname(arguments["settings_path"]) + + config_dict = arguments.copy() + ask_to_apply = config_dict.pop("ask_to_apply", False) + jobs = config_dict.pop("jobs", ()) + check = config_dict.pop("check", False) + show_diff = config_dict.pop("show_diff", False) + write_to_stdout = config_dict.pop("write_to_stdout", False) + deprecated_flags = config_dict.pop("deprecated_flags", False) + remapped_deprecated_args = config_dict.pop("remapped_deprecated_args", False) + wrong_sorted_files = False + all_attempt_broken = False + no_valid_encodings = False + + if "src_paths" in config_dict: + config_dict["src_paths"] = { + Path(src_path).resolve() for src_path in config_dict.get("src_paths", ()) + } + + config = Config(**config_dict) + if show_config: + print(json.dumps(config.__dict__, indent=4, separators=(",", ": "), default=_preconvert)) + return + elif file_names == ["-"]: + if show_files: + sys.exit("Error: can't show files for streaming input.") + + if check: + incorrectly_sorted = not api.check_stream( + input_stream=sys.stdin if stdin is None else stdin, + config=config, + show_diff=show_diff, + ) + + wrong_sorted_files = incorrectly_sorted + else: + api.sort_stream( + input_stream=sys.stdin if stdin is None else stdin, + output_stream=sys.stdout, + config=config, + show_diff=show_diff, + ) + else: + skipped: List[str] = [] + broken: List[str] = [] + + if config.filter_files: + filtered_files = [] + for file_name in file_names: + if config.is_skipped(Path(file_name)): + skipped.append(file_name) + else: + filtered_files.append(file_name) + file_names = filtered_files + + file_names = iter_source_code(file_names, config, skipped, broken) + if show_files: + for file_name in file_names: + print(file_name) + return + num_skipped = 0 + num_broken = 0 + num_invalid_encoding = 0 + if config.verbose: + print(ASCII_ART) + + if jobs: + import multiprocessing + + executor = multiprocessing.Pool(jobs) + attempt_iterator = executor.imap( + functools.partial( + sort_imports, + config=config, + check=check, + ask_to_apply=ask_to_apply, + write_to_stdout=write_to_stdout, + ), + file_names, + ) + else: + # https://github.com/python/typeshed/pull/2814 + attempt_iterator = ( + sort_imports( # type: ignore + file_name, + config=config, + check=check, + ask_to_apply=ask_to_apply, + show_diff=show_diff, + write_to_stdout=write_to_stdout, + ) + for file_name in file_names + ) + + # If any files passed in are missing considered as error, should be removed + is_no_attempt = True + any_encoding_valid = False + for sort_attempt in attempt_iterator: + if not sort_attempt: + continue # pragma: no cover - shouldn't happen, satisfies type constraint + incorrectly_sorted = sort_attempt.incorrectly_sorted + if arguments.get("check", False) and incorrectly_sorted: + wrong_sorted_files = True + if sort_attempt.skipped: + num_skipped += ( + 1 # pragma: no cover - shouldn't happen, due to skip in iter_source_code + ) + + if not sort_attempt.supported_encoding: + num_invalid_encoding += 1 + else: + any_encoding_valid = True + + is_no_attempt = False + + num_skipped += len(skipped) + if num_skipped and not arguments.get("quiet", False): + if config.verbose: + for was_skipped in skipped: + warn( + f"{was_skipped} was skipped as it's listed in 'skip' setting" + " or matches a glob in 'skip_glob' setting" + ) + print(f"Skipped {num_skipped} files") + + num_broken += len(broken) + if num_broken and not arguments.get("quite", False): + if config.verbose: + for was_broken in broken: + warn(f"{was_broken} was broken path, make sure it exists correctly") + print(f"Broken {num_broken} paths") + + if num_broken > 0 and is_no_attempt: + all_attempt_broken = True + if num_invalid_encoding > 0 and not any_encoding_valid: + no_valid_encodings = True + + if not config.quiet and (remapped_deprecated_args or deprecated_flags): + if remapped_deprecated_args: + warn( + "W0502: The following deprecated single dash CLI flags were used and translated: " + f"{', '.join(remapped_deprecated_args)}!" + ) + if deprecated_flags: + warn( + "W0501: The following deprecated CLI flags were used and ignored: " + f"{', '.join(deprecated_flags)}!" + ) + warn( + "W0500: Please see the 5.0.0 Upgrade guide: " + "https://pycqa.github.io/isort/docs/upgrade_guides/5.0.0/" + ) + + if wrong_sorted_files: + sys.exit(1) + + if all_attempt_broken: + sys.exit(1) + + if no_valid_encodings: + printer = create_terminal_printer(color=config.color_output) + printer.error("No valid encodings.") + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/WebCrawler/venv/Lib/site-packages/isort/output.py b/WebCrawler/venv/Lib/site-packages/isort/output.py new file mode 100644 index 0000000..d2633ff --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/isort/output.py @@ -0,0 +1,584 @@ +import copy +import itertools +from functools import partial +from typing import Iterable, List, Set, Tuple + +from isort.format import format_simplified + +from . import parse, sorting, wrap +from .comments import add_to_line as with_comments +from .settings import DEFAULT_CONFIG, Config + +STATEMENT_DECLARATIONS: Tuple[str, ...] = ("def ", "cdef ", "cpdef ", "class ", "@", "async def") + + +def sorted_imports( + parsed: parse.ParsedContent, + config: Config = DEFAULT_CONFIG, + extension: str = "py", + import_type: str = "import", +) -> str: + """Adds the imports back to the file. + + (at the index of the first import) sorted alphabetically and split between groups + + """ + if parsed.import_index == -1: + return _output_as_string(parsed.lines_without_imports, parsed.line_separator) + + formatted_output: List[str] = parsed.lines_without_imports.copy() + remove_imports = [format_simplified(removal) for removal in config.remove_imports] + + sections: Iterable[str] = itertools.chain(parsed.sections, config.forced_separate) + + if config.no_sections: + parsed.imports["no_sections"] = {"straight": {}, "from": {}} + base_sections: Tuple[str, ...] = () + for section in sections: + if section == "FUTURE": + base_sections = ("FUTURE",) + continue + parsed.imports["no_sections"]["straight"].update( + parsed.imports[section].get("straight", {}) + ) + parsed.imports["no_sections"]["from"].update(parsed.imports[section].get("from", {})) + sections = base_sections + ("no_sections",) + + output: List[str] = [] + seen_headings: Set[str] = set() + pending_lines_before = False + for section in sections: + straight_modules = parsed.imports[section]["straight"] + if not config.only_sections: + straight_modules = sorting.naturally( + straight_modules, + key=lambda key: sorting.module_key( + key, config, section_name=section, straight_import=True + ), + ) + + from_modules = parsed.imports[section]["from"] + if not config.only_sections: + from_modules = sorting.naturally( + from_modules, key=lambda key: sorting.module_key(key, config, section_name=section) + ) + + straight_imports = _with_straight_imports( + parsed, config, straight_modules, section, remove_imports, import_type + ) + from_imports = _with_from_imports( + parsed, config, from_modules, section, remove_imports, import_type + ) + + lines_between = [""] * ( + config.lines_between_types if from_modules and straight_modules else 0 + ) + if config.from_first: + section_output = from_imports + lines_between + straight_imports + else: + section_output = straight_imports + lines_between + from_imports + + if config.force_sort_within_sections: + # collapse comments + comments_above = [] + new_section_output: List[str] = [] + for line in section_output: + if not line: + continue + if line.startswith("#"): + comments_above.append(line) + elif comments_above: + new_section_output.append(_LineWithComments(line, comments_above)) + comments_above = [] + else: + new_section_output.append(line) + # only_sections options is not imposed if force_sort_within_sections is True + new_section_output = sorting.naturally( + new_section_output, + key=partial( + sorting.section_key, + order_by_type=config.order_by_type, + force_to_top=config.force_to_top, + lexicographical=config.lexicographical, + length_sort=config.length_sort, + reverse_relative=config.reverse_relative, + group_by_package=config.group_by_package, + ), + ) + + # uncollapse comments + section_output = [] + for line in new_section_output: + comments = getattr(line, "comments", ()) + if comments: + section_output.extend(comments) + section_output.append(str(line)) + + section_name = section + no_lines_before = section_name in config.no_lines_before + + if section_output: + if section_name in parsed.place_imports: + parsed.place_imports[section_name] = section_output + continue + + section_title = config.import_headings.get(section_name.lower(), "") + if section_title and section_title not in seen_headings: + if config.dedup_headings: + seen_headings.add(section_title) + section_comment = f"# {section_title}" + if section_comment not in parsed.lines_without_imports[0:1]: + section_output.insert(0, section_comment) + + if pending_lines_before or not no_lines_before: + output += [""] * config.lines_between_sections + + output += section_output + + pending_lines_before = False + else: + pending_lines_before = pending_lines_before or not no_lines_before + + if config.ensure_newline_before_comments: + output = _ensure_newline_before_comment(output) + + while output and output[-1].strip() == "": + output.pop() # pragma: no cover + while output and output[0].strip() == "": + output.pop(0) + + if config.formatting_function: + output = config.formatting_function( + parsed.line_separator.join(output), extension, config + ).splitlines() + + output_at = 0 + if parsed.import_index < parsed.original_line_count: + output_at = parsed.import_index + formatted_output[output_at:0] = output + + imports_tail = output_at + len(output) + while [ + character.strip() for character in formatted_output[imports_tail : imports_tail + 1] + ] == [""]: + formatted_output.pop(imports_tail) + + if len(formatted_output) > imports_tail: + next_construct = "" + tail = formatted_output[imports_tail:] + + for index, line in enumerate(tail): + should_skip, in_quote, *_ = parse.skip_line( + line, + in_quote="", + index=len(formatted_output), + section_comments=config.section_comments, + needs_import=False, + ) + if not should_skip and line.strip(): + if ( + line.strip().startswith("#") + and len(tail) > (index + 1) + and tail[index + 1].strip() + ): + continue + next_construct = line + break + elif in_quote: + next_construct = line + break + + if config.lines_after_imports != -1: + formatted_output[imports_tail:0] = ["" for line in range(config.lines_after_imports)] + elif extension != "pyi" and next_construct.startswith(STATEMENT_DECLARATIONS): + formatted_output[imports_tail:0] = ["", ""] + else: + formatted_output[imports_tail:0] = [""] + + if parsed.place_imports: + new_out_lines = [] + for index, line in enumerate(formatted_output): + new_out_lines.append(line) + if line in parsed.import_placements: + new_out_lines.extend(parsed.place_imports[parsed.import_placements[line]]) + if ( + len(formatted_output) <= (index + 1) + or formatted_output[index + 1].strip() != "" + ): + new_out_lines.append("") + formatted_output = new_out_lines + + return _output_as_string(formatted_output, parsed.line_separator) + + +def _with_from_imports( + parsed: parse.ParsedContent, + config: Config, + from_modules: Iterable[str], + section: str, + remove_imports: List[str], + import_type: str, +) -> List[str]: + output: List[str] = [] + for module in from_modules: + if module in remove_imports: + continue + + import_start = f"from {module} {import_type} " + from_imports = list(parsed.imports[section]["from"][module]) + if not config.no_inline_sort or ( + config.force_single_line and module not in config.single_line_exclusions + ): + ignore_case = config.force_alphabetical_sort_within_sections + + if not config.only_sections: + from_imports = sorting.naturally( + from_imports, + key=lambda key: sorting.module_key( + key, + config, + True, + ignore_case, + section_name=section, + ), + ) + if remove_imports: + from_imports = [ + line for line in from_imports if f"{module}.{line}" not in remove_imports + ] + + sub_modules = [f"{module}.{from_import}" for from_import in from_imports] + as_imports = { + from_import: [ + f"{from_import} as {as_module}" for as_module in parsed.as_map["from"][sub_module] + ] + for from_import, sub_module in zip(from_imports, sub_modules) + if sub_module in parsed.as_map["from"] + } + if config.combine_as_imports and not ("*" in from_imports and config.combine_star): + if not config.no_inline_sort: + for as_import in as_imports: + if not config.only_sections: + as_imports[as_import] = sorting.naturally(as_imports[as_import]) + for from_import in copy.copy(from_imports): + if from_import in as_imports: + idx = from_imports.index(from_import) + if parsed.imports[section]["from"][module][from_import]: + from_imports[(idx + 1) : (idx + 1)] = as_imports.pop(from_import) + else: + from_imports[idx : (idx + 1)] = as_imports.pop(from_import) + + only_show_as_imports = False + comments = parsed.categorized_comments["from"].pop(module, ()) + above_comments = parsed.categorized_comments["above"]["from"].pop(module, None) + while from_imports: + if above_comments: + output.extend(above_comments) + above_comments = None + + if "*" in from_imports and config.combine_star: + import_statement = wrap.line( + with_comments( + _with_star_comments(parsed, module, list(comments or ())), + f"{import_start}*", + removed=config.ignore_comments, + comment_prefix=config.comment_prefix, + ), + parsed.line_separator, + config, + ) + from_imports = [ + from_import for from_import in from_imports if from_import in as_imports + ] + only_show_as_imports = True + elif config.force_single_line and module not in config.single_line_exclusions: + import_statement = "" + while from_imports: + from_import = from_imports.pop(0) + single_import_line = with_comments( + comments, + import_start + from_import, + removed=config.ignore_comments, + comment_prefix=config.comment_prefix, + ) + comment = ( + parsed.categorized_comments["nested"].get(module, {}).pop(from_import, None) + ) + if comment: + single_import_line += ( + f"{comments and ';' or config.comment_prefix} " f"{comment}" + ) + if from_import in as_imports: + if ( + parsed.imports[section]["from"][module][from_import] + and not only_show_as_imports + ): + output.append( + wrap.line(single_import_line, parsed.line_separator, config) + ) + from_comments = parsed.categorized_comments["straight"].get( + f"{module}.{from_import}" + ) + + if not config.only_sections: + output.extend( + with_comments( + from_comments, + wrap.line( + import_start + as_import, parsed.line_separator, config + ), + removed=config.ignore_comments, + comment_prefix=config.comment_prefix, + ) + for as_import in sorting.naturally(as_imports[from_import]) + ) + + else: + output.extend( + with_comments( + from_comments, + wrap.line( + import_start + as_import, parsed.line_separator, config + ), + removed=config.ignore_comments, + comment_prefix=config.comment_prefix, + ) + for as_import in as_imports[from_import] + ) + else: + output.append(wrap.line(single_import_line, parsed.line_separator, config)) + comments = None + else: + while from_imports and from_imports[0] in as_imports: + from_import = from_imports.pop(0) + + if not config.only_sections: + as_imports[from_import] = sorting.naturally(as_imports[from_import]) + from_comments = ( + parsed.categorized_comments["straight"].get(f"{module}.{from_import}") or [] + ) + if ( + parsed.imports[section]["from"][module][from_import] + and not only_show_as_imports + ): + specific_comment = ( + parsed.categorized_comments["nested"] + .get(module, {}) + .pop(from_import, None) + ) + if specific_comment: + from_comments.append(specific_comment) + output.append( + wrap.line( + with_comments( + from_comments, + import_start + from_import, + removed=config.ignore_comments, + comment_prefix=config.comment_prefix, + ), + parsed.line_separator, + config, + ) + ) + from_comments = [] + + for as_import in as_imports[from_import]: + specific_comment = ( + parsed.categorized_comments["nested"] + .get(module, {}) + .pop(as_import, None) + ) + if specific_comment: + from_comments.append(specific_comment) + + output.append( + wrap.line( + with_comments( + from_comments, + import_start + as_import, + removed=config.ignore_comments, + comment_prefix=config.comment_prefix, + ), + parsed.line_separator, + config, + ) + ) + + from_comments = [] + + if "*" in from_imports: + output.append( + with_comments( + _with_star_comments(parsed, module, list(comments or ())), + f"{import_start}*", + removed=config.ignore_comments, + comment_prefix=config.comment_prefix, + ) + ) + from_imports.remove("*") + comments = None + + for from_import in copy.copy(from_imports): + comment = ( + parsed.categorized_comments["nested"].get(module, {}).pop(from_import, None) + ) + if comment: + single_import_line = with_comments( + comments, + import_start + from_import, + removed=config.ignore_comments, + comment_prefix=config.comment_prefix, + ) + single_import_line += ( + f"{comments and ';' or config.comment_prefix} " f"{comment}" + ) + output.append(wrap.line(single_import_line, parsed.line_separator, config)) + from_imports.remove(from_import) + comments = None + + from_import_section = [] + while from_imports and ( + from_imports[0] not in as_imports + or ( + config.combine_as_imports + and parsed.imports[section]["from"][module][from_import] + ) + ): + from_import_section.append(from_imports.pop(0)) + if config.combine_as_imports: + comments = (comments or []) + list( + parsed.categorized_comments["from"].pop(f"{module}.__combined_as__", ()) + ) + import_statement = with_comments( + comments, + import_start + (", ").join(from_import_section), + removed=config.ignore_comments, + comment_prefix=config.comment_prefix, + ) + if not from_import_section: + import_statement = "" + + do_multiline_reformat = False + + force_grid_wrap = config.force_grid_wrap + if force_grid_wrap and len(from_import_section) >= force_grid_wrap: + do_multiline_reformat = True + + if len(import_statement) > config.line_length and len(from_import_section) > 1: + do_multiline_reformat = True + + # If line too long AND have imports AND we are + # NOT using GRID or VERTICAL wrap modes + if ( + len(import_statement) > config.line_length + and len(from_import_section) > 0 + and config.multi_line_output + not in (wrap.Modes.GRID, wrap.Modes.VERTICAL) # type: ignore + ): + do_multiline_reformat = True + + if do_multiline_reformat: + import_statement = wrap.import_statement( + import_start=import_start, + from_imports=from_import_section, + comments=comments, + line_separator=parsed.line_separator, + config=config, + ) + if config.multi_line_output == wrap.Modes.GRID: # type: ignore + other_import_statement = wrap.import_statement( + import_start=import_start, + from_imports=from_import_section, + comments=comments, + line_separator=parsed.line_separator, + config=config, + multi_line_output=wrap.Modes.VERTICAL_GRID, # type: ignore + ) + if max(len(x) for x in import_statement.split("\n")) > config.line_length: + import_statement = other_import_statement + if not do_multiline_reformat and len(import_statement) > config.line_length: + import_statement = wrap.line(import_statement, parsed.line_separator, config) + + if import_statement: + output.append(import_statement) + return output + + +def _with_straight_imports( + parsed: parse.ParsedContent, + config: Config, + straight_modules: Iterable[str], + section: str, + remove_imports: List[str], + import_type: str, +) -> List[str]: + output: List[str] = [] + for module in straight_modules: + if module in remove_imports: + continue + + import_definition = [] + if module in parsed.as_map["straight"]: + if parsed.imports[section]["straight"][module]: + import_definition.append(f"{import_type} {module}") + import_definition.extend( + f"{import_type} {module} as {as_import}" + for as_import in parsed.as_map["straight"][module] + ) + else: + import_definition.append(f"{import_type} {module}") + + comments_above = parsed.categorized_comments["above"]["straight"].pop(module, None) + if comments_above: + output.extend(comments_above) + output.extend( + with_comments( + parsed.categorized_comments["straight"].get(module), + idef, + removed=config.ignore_comments, + comment_prefix=config.comment_prefix, + ) + for idef in import_definition + ) + + return output + + +def _output_as_string(lines: List[str], line_separator: str) -> str: + return line_separator.join(_normalize_empty_lines(lines)) + + +def _normalize_empty_lines(lines: List[str]) -> List[str]: + while lines and lines[-1].strip() == "": + lines.pop(-1) + + lines.append("") + return lines + + +class _LineWithComments(str): + def __new__(cls, value, comments): + instance = super().__new__(cls, value) # type: ignore + instance.comments = comments + return instance + + +def _ensure_newline_before_comment(output): + new_output: List[str] = [] + + def is_comment(line): + return line and line.startswith("#") + + for line, prev_line in zip(output, [None] + output): + if is_comment(line) and prev_line != "" and not is_comment(prev_line): + new_output.append("") + new_output.append(line) + return new_output + + +def _with_star_comments(parsed: parse.ParsedContent, module: str, comments: List[str]) -> List[str]: + star_comment = parsed.categorized_comments["nested"].get(module, {}).pop("*", None) + if star_comment: + return comments + [star_comment] + else: + return comments diff --git a/WebCrawler/venv/Lib/site-packages/isort/parse.py b/WebCrawler/venv/Lib/site-packages/isort/parse.py new file mode 100644 index 0000000..fe3dd15 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/isort/parse.py @@ -0,0 +1,548 @@ +"""Defines parsing functions used by isort for parsing import definitions""" +from collections import OrderedDict, defaultdict +from functools import partial +from itertools import chain +from typing import TYPE_CHECKING, Any, Dict, List, NamedTuple, Optional, Tuple +from warnings import warn + +from . import place +from .comments import parse as parse_comments +from .deprecated.finders import FindersManager +from .settings import DEFAULT_CONFIG, Config + +if TYPE_CHECKING: + from mypy_extensions import TypedDict + + CommentsAboveDict = TypedDict( + "CommentsAboveDict", {"straight": Dict[str, Any], "from": Dict[str, Any]} + ) + + CommentsDict = TypedDict( + "CommentsDict", + { + "from": Dict[str, Any], + "straight": Dict[str, Any], + "nested": Dict[str, Any], + "above": CommentsAboveDict, + }, + ) + + +def _infer_line_separator(contents: str) -> str: + if "\r\n" in contents: + return "\r\n" + elif "\r" in contents: + return "\r" + else: + return "\n" + + +def _normalize_line(raw_line: str) -> Tuple[str, str]: + """Normalizes import related statements in the provided line. + + Returns (normalized_line: str, raw_line: str) + """ + line = raw_line.replace("from.import ", "from . import ") + line = line.replace("from.cimport ", "from . cimport ") + line = line.replace("import*", "import *") + line = line.replace(" .import ", " . import ") + line = line.replace(" .cimport ", " . cimport ") + line = line.replace("\t", " ") + return (line, raw_line) + + +def import_type(line: str, config: Config = DEFAULT_CONFIG) -> Optional[str]: + """If the current line is an import line it will return its type (from or straight)""" + if config.honor_noqa and line.lower().rstrip().endswith("noqa"): + return None + elif "isort:skip" in line or "isort: skip" in line or "isort: split" in line: + return None + elif line.startswith(("import ", "cimport ")): + return "straight" + elif line.startswith("from "): + return "from" + return None + + +def _strip_syntax(import_string: str) -> str: + import_string = import_string.replace("_import", "[[i]]") + import_string = import_string.replace("_cimport", "[[ci]]") + for remove_syntax in ["\\", "(", ")", ","]: + import_string = import_string.replace(remove_syntax, " ") + import_list = import_string.split() + for key in ("from", "import", "cimport"): + if key in import_list: + import_list.remove(key) + import_string = " ".join(import_list) + import_string = import_string.replace("[[i]]", "_import") + import_string = import_string.replace("[[ci]]", "_cimport") + return import_string.replace("{ ", "{|").replace(" }", "|}") + + +def skip_line( + line: str, + in_quote: str, + index: int, + section_comments: Tuple[str, ...], + needs_import: bool = True, +) -> Tuple[bool, str]: + """Determine if a given line should be skipped. + + Returns back a tuple containing: + + (skip_line: bool, + in_quote: str,) + """ + should_skip = bool(in_quote) + if '"' in line or "'" in line: + char_index = 0 + while char_index < len(line): + if line[char_index] == "\\": + char_index += 1 + elif in_quote: + if line[char_index : char_index + len(in_quote)] == in_quote: + in_quote = "" + elif line[char_index] in ("'", '"'): + long_quote = line[char_index : char_index + 3] + if long_quote in ('"""', "'''"): + in_quote = long_quote + char_index += 2 + else: + in_quote = line[char_index] + elif line[char_index] == "#": + break + char_index += 1 + + if ";" in line.split("#")[0] and needs_import: + for part in (part.strip() for part in line.split(";")): + if ( + part + and not part.startswith("from ") + and not part.startswith(("import ", "cimport ")) + ): + should_skip = True + + return (bool(should_skip or in_quote), in_quote) + + +class ParsedContent(NamedTuple): + in_lines: List[str] + lines_without_imports: List[str] + import_index: int + place_imports: Dict[str, List[str]] + import_placements: Dict[str, str] + as_map: Dict[str, Dict[str, List[str]]] + imports: Dict[str, Dict[str, Any]] + categorized_comments: "CommentsDict" + change_count: int + original_line_count: int + line_separator: str + sections: Any + verbose_output: List[str] + + +def file_contents(contents: str, config: Config = DEFAULT_CONFIG) -> ParsedContent: + """Parses a python file taking out and categorizing imports.""" + line_separator: str = config.line_ending or _infer_line_separator(contents) + in_lines = contents.splitlines() + if contents and contents[-1] in ("\n", "\r"): + in_lines.append("") + + out_lines = [] + original_line_count = len(in_lines) + if config.old_finders: + finder = FindersManager(config=config).find + else: + finder = partial(place.module, config=config) + + line_count = len(in_lines) + + place_imports: Dict[str, List[str]] = {} + import_placements: Dict[str, str] = {} + as_map: Dict[str, Dict[str, List[str]]] = { + "straight": defaultdict(list), + "from": defaultdict(list), + } + imports: OrderedDict[str, Dict[str, Any]] = OrderedDict() + verbose_output: List[str] = [] + + for section in chain(config.sections, config.forced_separate): + imports[section] = {"straight": OrderedDict(), "from": OrderedDict()} + categorized_comments: CommentsDict = { + "from": {}, + "straight": {}, + "nested": {}, + "above": {"straight": {}, "from": {}}, + } + + index = 0 + import_index = -1 + in_quote = "" + while index < line_count: + line = in_lines[index] + index += 1 + statement_index = index + (skipping_line, in_quote) = skip_line( + line, in_quote=in_quote, index=index, section_comments=config.section_comments + ) + + if line in config.section_comments and not skipping_line: + if import_index == -1: + import_index = index - 1 + continue + + if "isort:imports-" in line and line.startswith("#"): + section = line.split("isort:imports-")[-1].split()[0].upper() + place_imports[section] = [] + import_placements[line] = section + elif "isort: imports-" in line and line.startswith("#"): + section = line.split("isort: imports-")[-1].split()[0].upper() + place_imports[section] = [] + import_placements[line] = section + + if skipping_line: + out_lines.append(line) + continue + + lstripped_line = line.lstrip() + if ( + config.float_to_top + and import_index == -1 + and line + and not in_quote + and not lstripped_line.startswith("#") + and not lstripped_line.startswith("'''") + and not lstripped_line.startswith('"""') + ): + if not lstripped_line.startswith("import") and not lstripped_line.startswith("from"): + import_index = index - 1 + while import_index and not in_lines[import_index - 1]: + import_index -= 1 + else: + commentless = line.split("#", 1)[0].strip() + if ( + ("isort:skip" in line or "isort: skip" in line) + and "(" in commentless + and ")" not in commentless + ): + import_index = index + + starting_line = line + while "isort:skip" in starting_line or "isort: skip" in starting_line: + commentless = starting_line.split("#", 1)[0] + if ( + "(" in commentless + and not commentless.rstrip().endswith(")") + and import_index < line_count + ): + + while import_index < line_count and not commentless.rstrip().endswith( + ")" + ): + commentless = in_lines[import_index].split("#", 1)[0] + import_index += 1 + else: + import_index += 1 + + if import_index >= line_count: + break + else: + starting_line = in_lines[import_index] + + line, *end_of_line_comment = line.split("#", 1) + if ";" in line: + statements = [line.strip() for line in line.split(";")] + else: + statements = [line] + if end_of_line_comment: + statements[-1] = f"{statements[-1]}#{end_of_line_comment[0]}" + + for statement in statements: + line, raw_line = _normalize_line(statement) + type_of_import = import_type(line, config) or "" + if not type_of_import: + out_lines.append(raw_line) + continue + + if import_index == -1: + import_index = index - 1 + nested_comments = {} + import_string, comment = parse_comments(line) + comments = [comment] if comment else [] + line_parts = [part for part in _strip_syntax(import_string).strip().split(" ") if part] + if type_of_import == "from" and len(line_parts) == 2 and comments: + nested_comments[line_parts[-1]] = comments[0] + + if "(" in line.split("#", 1)[0] and index < line_count: + while not line.split("#")[0].strip().endswith(")") and index < line_count: + line, new_comment = parse_comments(in_lines[index]) + index += 1 + if new_comment: + comments.append(new_comment) + stripped_line = _strip_syntax(line).strip() + if ( + type_of_import == "from" + and stripped_line + and " " not in stripped_line.replace(" as ", "") + and new_comment + ): + nested_comments[stripped_line] = comments[-1] + import_string += line_separator + line + else: + while line.strip().endswith("\\"): + line, new_comment = parse_comments(in_lines[index]) + index += 1 + if new_comment: + comments.append(new_comment) + + # Still need to check for parentheses after an escaped line + if ( + "(" in line.split("#")[0] + and ")" not in line.split("#")[0] + and index < line_count + ): + stripped_line = _strip_syntax(line).strip() + if ( + type_of_import == "from" + and stripped_line + and " " not in stripped_line.replace(" as ", "") + and new_comment + ): + nested_comments[stripped_line] = comments[-1] + import_string += line_separator + line + + while not line.split("#")[0].strip().endswith(")") and index < line_count: + line, new_comment = parse_comments(in_lines[index]) + index += 1 + if new_comment: + comments.append(new_comment) + stripped_line = _strip_syntax(line).strip() + if ( + type_of_import == "from" + and stripped_line + and " " not in stripped_line.replace(" as ", "") + and new_comment + ): + nested_comments[stripped_line] = comments[-1] + import_string += line_separator + line + + stripped_line = _strip_syntax(line).strip() + if ( + type_of_import == "from" + and stripped_line + and " " not in stripped_line.replace(" as ", "") + and new_comment + ): + nested_comments[stripped_line] = comments[-1] + if import_string.strip().endswith( + (" import", " cimport") + ) or line.strip().startswith(("import ", "cimport ")): + import_string += line_separator + line + else: + import_string = import_string.rstrip().rstrip("\\") + " " + line.lstrip() + + if type_of_import == "from": + cimports: bool + import_string = ( + import_string.replace("import(", "import (") + .replace("\\", " ") + .replace("\n", " ") + ) + if " cimport " in import_string: + parts = import_string.split(" cimport ") + cimports = True + + else: + parts = import_string.split(" import ") + cimports = False + + from_import = parts[0].split(" ") + import_string = (" cimport " if cimports else " import ").join( + [from_import[0] + " " + "".join(from_import[1:])] + parts[1:] + ) + + just_imports = [ + item.replace("{|", "{ ").replace("|}", " }") + for item in _strip_syntax(import_string).split() + ] + + attach_comments_to: Optional[List[Any]] = None + direct_imports = just_imports[1:] + straight_import = True + if "as" in just_imports and (just_imports.index("as") + 1) < len(just_imports): + straight_import = False + while "as" in just_imports: + nested_module = None + as_index = just_imports.index("as") + if type_of_import == "from": + nested_module = just_imports[as_index - 1] + top_level_module = just_imports[0] + module = top_level_module + "." + nested_module + as_name = just_imports[as_index + 1] + direct_imports.remove(nested_module) + direct_imports.remove(as_name) + direct_imports.remove("as") + if nested_module == as_name and config.remove_redundant_aliases: + pass + elif as_name not in as_map["from"][module]: + as_map["from"][module].append(as_name) + + full_name = f"{nested_module} as {as_name}" + associated_comment = nested_comments.get(full_name) + if associated_comment: + categorized_comments["nested"].setdefault(top_level_module, {})[ + full_name + ] = associated_comment + if associated_comment in comments: + comments.pop(comments.index(associated_comment)) + else: + module = just_imports[as_index - 1] + as_name = just_imports[as_index + 1] + if module == as_name and config.remove_redundant_aliases: + pass + elif as_name not in as_map["straight"][module]: + as_map["straight"][module].append(as_name) + + if comments and attach_comments_to is None: + if nested_module and config.combine_as_imports: + attach_comments_to = categorized_comments["from"].setdefault( + f"{top_level_module}.__combined_as__", [] + ) + else: + attach_comments_to = categorized_comments["straight"].setdefault( + module, [] + ) + del just_imports[as_index : as_index + 2] + + if type_of_import == "from": + import_from = just_imports.pop(0) + placed_module = finder(import_from) + if config.verbose and not config.only_modified: + print(f"from-type place_module for {import_from} returned {placed_module}") + + elif config.verbose: + verbose_output.append( + f"from-type place_module for {import_from} returned {placed_module}" + ) + if placed_module == "": + warn( + f"could not place module {import_from} of line {line} --" + " Do you need to define a default section?" + ) + root = imports[placed_module][type_of_import] # type: ignore + for import_name in just_imports: + associated_comment = nested_comments.get(import_name) + if associated_comment: + categorized_comments["nested"].setdefault(import_from, {})[ + import_name + ] = associated_comment + if associated_comment in comments: + comments.pop(comments.index(associated_comment)) + + if comments and attach_comments_to is None: + attach_comments_to = categorized_comments["from"].setdefault(import_from, []) + + if len(out_lines) > max(import_index, 1) - 1: + last = out_lines and out_lines[-1].rstrip() or "" + while ( + last.startswith("#") + and not last.endswith('"""') + and not last.endswith("'''") + and "isort:imports-" not in last + and "isort: imports-" not in last + and not config.treat_all_comments_as_code + and not last.strip() in config.treat_comments_as_code + ): + categorized_comments["above"]["from"].setdefault(import_from, []).insert( + 0, out_lines.pop(-1) + ) + if out_lines: + last = out_lines[-1].rstrip() + else: + last = "" + if statement_index - 1 == import_index: # pragma: no cover + import_index -= len( + categorized_comments["above"]["from"].get(import_from, []) + ) + + if import_from not in root: + root[import_from] = OrderedDict( + (module, module in direct_imports) for module in just_imports + ) + else: + root[import_from].update( + (module, root[import_from].get(module, False) or module in direct_imports) + for module in just_imports + ) + + if comments and attach_comments_to is not None: + attach_comments_to.extend(comments) + else: + if comments and attach_comments_to is not None: + attach_comments_to.extend(comments) + comments = [] + + for module in just_imports: + if comments: + categorized_comments["straight"][module] = comments + comments = [] + + if len(out_lines) > max(import_index, +1, 1) - 1: + + last = out_lines and out_lines[-1].rstrip() or "" + while ( + last.startswith("#") + and not last.endswith('"""') + and not last.endswith("'''") + and "isort:imports-" not in last + and "isort: imports-" not in last + and not config.treat_all_comments_as_code + and not last.strip() in config.treat_comments_as_code + ): + categorized_comments["above"]["straight"].setdefault(module, []).insert( + 0, out_lines.pop(-1) + ) + if out_lines: + last = out_lines[-1].rstrip() + else: + last = "" + if index - 1 == import_index: + import_index -= len( + categorized_comments["above"]["straight"].get(module, []) + ) + placed_module = finder(module) + if config.verbose and not config.only_modified: + print(f"else-type place_module for {module} returned {placed_module}") + + elif config.verbose: + verbose_output.append( + f"else-type place_module for {module} returned {placed_module}" + ) + if placed_module == "": + warn( + f"could not place module {module} of line {line} --" + " Do you need to define a default section?" + ) + imports.setdefault("", {"straight": OrderedDict(), "from": OrderedDict()}) + straight_import |= imports[placed_module][type_of_import].get( # type: ignore + module, False + ) + imports[placed_module][type_of_import][module] = straight_import # type: ignore + + change_count = len(out_lines) - original_line_count + + return ParsedContent( + in_lines=in_lines, + lines_without_imports=out_lines, + import_index=import_index, + place_imports=place_imports, + import_placements=import_placements, + as_map=as_map, + imports=imports, + categorized_comments=categorized_comments, + change_count=change_count, + original_line_count=original_line_count, + line_separator=line_separator, + sections=config.sections, + verbose_output=verbose_output, + ) diff --git a/WebCrawler/venv/Lib/site-packages/isort/place.py b/WebCrawler/venv/Lib/site-packages/isort/place.py new file mode 100644 index 0000000..aaf954a --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/isort/place.py @@ -0,0 +1,145 @@ +"""Contains all logic related to placing an import within a certain section.""" +import importlib +from fnmatch import fnmatch +from functools import lru_cache +from pathlib import Path +from typing import FrozenSet, Iterable, Optional, Tuple + +from isort import sections +from isort.settings import DEFAULT_CONFIG, Config +from isort.utils import exists_case_sensitive + +LOCAL = "LOCALFOLDER" + + +def module(name: str, config: Config = DEFAULT_CONFIG) -> str: + """Returns the section placement for the given module name.""" + return module_with_reason(name, config)[0] + + +@lru_cache(maxsize=1000) +def module_with_reason(name: str, config: Config = DEFAULT_CONFIG) -> Tuple[str, str]: + """Returns the section placement for the given module name alongside the reasoning.""" + return ( + _forced_separate(name, config) + or _local(name, config) + or _known_pattern(name, config) + or _src_path(name, config) + or (config.default_section, "Default option in Config or universal default.") + ) + + +def _forced_separate(name: str, config: Config) -> Optional[Tuple[str, str]]: + for forced_separate in config.forced_separate: + # Ensure all forced_separate patterns will match to end of string + path_glob = forced_separate + if not forced_separate.endswith("*"): + path_glob = "%s*" % forced_separate + + if fnmatch(name, path_glob) or fnmatch(name, "." + path_glob): + return (forced_separate, f"Matched forced_separate ({forced_separate}) config value.") + + return None + + +def _local(name: str, config: Config) -> Optional[Tuple[str, str]]: + if name.startswith("."): + return (LOCAL, "Module name started with a dot.") + + return None + + +def _known_pattern(name: str, config: Config) -> Optional[Tuple[str, str]]: + parts = name.split(".") + module_names_to_check = (".".join(parts[:first_k]) for first_k in range(len(parts), 0, -1)) + for module_name_to_check in module_names_to_check: + for pattern, placement in config.known_patterns: + if placement in config.sections and pattern.match(module_name_to_check): + return (placement, f"Matched configured known pattern {pattern}") + + return None + + +def _src_path( + name: str, + config: Config, + src_paths: Optional[Iterable[Path]] = None, + prefix: Tuple[str, ...] = (), +) -> Optional[Tuple[str, str]]: + if src_paths is None: + src_paths = config.src_paths + + root_module_name, *nested_module = name.split(".", 1) + new_prefix = prefix + (root_module_name,) + namespace = ".".join(new_prefix) + + for src_path in src_paths: + module_path = (src_path / root_module_name).resolve() + if not prefix and not module_path.is_dir() and src_path.name == root_module_name: + module_path = src_path.resolve() + if nested_module and ( + namespace in config.namespace_packages + or ( + config.auto_identify_namespace_packages + and _is_namespace_package(module_path, config.supported_extensions) + ) + ): + return _src_path(nested_module[0], config, (module_path,), new_prefix) + if ( + _is_module(module_path) + or _is_package(module_path) + or _src_path_is_module(src_path, root_module_name) + ): + return (sections.FIRSTPARTY, f"Found in one of the configured src_paths: {src_path}.") + + return None + + +def _is_module(path: Path) -> bool: + return ( + exists_case_sensitive(str(path.with_suffix(".py"))) + or any( + exists_case_sensitive(str(path.with_suffix(ext_suffix))) + for ext_suffix in importlib.machinery.EXTENSION_SUFFIXES + ) + or exists_case_sensitive(str(path / "__init__.py")) + ) + + +def _is_package(path: Path) -> bool: + return exists_case_sensitive(str(path)) and path.is_dir() + + +def _is_namespace_package(path: Path, src_extensions: FrozenSet[str]) -> bool: + if not _is_package(path): + return False + + init_file = path / "__init__.py" + if not init_file.exists(): + filenames = [ + filepath + for filepath in path.iterdir() + if filepath.suffix.lstrip(".") in src_extensions + or filepath.name.lower() in ("setup.cfg", "pyproject.toml") + ] + if filenames: + return False + else: + with init_file.open() as open_init_file: + file_start = open_init_file.read(4096) + if ( + "__import__('pkg_resources').declare_namespace(__name__)" not in file_start + and '__import__("pkg_resources").declare_namespace(__name__)' not in file_start + and "__path__ = __import__('pkgutil').extend_path(__path__, __name__)" + not in file_start + and '__path__ = __import__("pkgutil").extend_path(__path__, __name__)' + not in file_start + ): + return False + return True + + +def _src_path_is_module(src_path: Path, module_name: str) -> bool: + return ( + module_name == src_path.name and src_path.is_dir() and exists_case_sensitive(str(src_path)) + ) diff --git a/WebCrawler/venv/Lib/site-packages/isort/profiles.py b/WebCrawler/venv/Lib/site-packages/isort/profiles.py new file mode 100644 index 0000000..cb8cb56 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/isort/profiles.py @@ -0,0 +1,68 @@ +"""Common profiles are defined here to be easily used within a project using --profile {name}""" +from typing import Any, Dict + +black = { + "multi_line_output": 3, + "include_trailing_comma": True, + "force_grid_wrap": 0, + "use_parentheses": True, + "ensure_newline_before_comments": True, + "line_length": 88, +} +django = { + "combine_as_imports": True, + "include_trailing_comma": True, + "multi_line_output": 5, + "line_length": 79, +} +pycharm = { + "multi_line_output": 3, + "force_grid_wrap": 2, + "lines_after_imports": 2, +} +google = { + "force_single_line": True, + "force_sort_within_sections": True, + "lexicographical": True, + "single_line_exclusions": ("typing",), + "order_by_type": False, + "group_by_package": True, +} +open_stack = { + "force_single_line": True, + "force_sort_within_sections": True, + "lexicographical": True, +} +plone = { + "force_alphabetical_sort": True, + "force_single_line": True, + "lines_after_imports": 2, + "line_length": 200, +} +attrs = { + "atomic": True, + "force_grid_wrap": 0, + "include_trailing_comma": True, + "lines_after_imports": 2, + "lines_between_types": 1, + "multi_line_output": 3, + "use_parentheses": True, +} +hug = { + "multi_line_output": 3, + "include_trailing_comma": True, + "force_grid_wrap": 0, + "use_parentheses": True, + "line_length": 100, +} + +profiles: Dict[str, Dict[str, Any]] = { + "black": black, + "django": django, + "pycharm": pycharm, + "google": google, + "open_stack": open_stack, + "plone": plone, + "attrs": attrs, + "hug": hug, +} diff --git a/WebCrawler/venv/Lib/site-packages/isort/pylama_isort.py b/WebCrawler/venv/Lib/site-packages/isort/pylama_isort.py new file mode 100644 index 0000000..4d4ffb9 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/isort/pylama_isort.py @@ -0,0 +1,43 @@ +import os +import sys +from contextlib import contextmanager +from typing import Any, Dict, List + +from pylama.lint import Linter as BaseLinter + +from isort.exceptions import FileSkipped + +from . import api + + +@contextmanager +def supress_stdout(): + stdout = sys.stdout + with open(os.devnull, "w") as devnull: + sys.stdout = devnull + yield + sys.stdout = stdout + + +class Linter(BaseLinter): + def allow(self, path: str) -> bool: + """Determine if this path should be linted.""" + return path.endswith(".py") + + def run(self, path: str, **meta: Any) -> List[Dict[str, Any]]: + """Lint the file. Return an array of error dicts if appropriate.""" + with supress_stdout(): + try: + if not api.check_file(path, disregard_skip=False): + return [ + { + "lnum": 0, + "col": 0, + "text": "Incorrectly sorted imports.", + "type": "ISORT", + } + ] + except FileSkipped: + pass + + return [] diff --git a/WebCrawler/venv/Lib/site-packages/isort/sections.py b/WebCrawler/venv/Lib/site-packages/isort/sections.py new file mode 100644 index 0000000..f59db69 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/isort/sections.py @@ -0,0 +1,9 @@ +"""Defines all sections isort uses by default""" +from typing import Tuple + +FUTURE: str = "FUTURE" +STDLIB: str = "STDLIB" +THIRDPARTY: str = "THIRDPARTY" +FIRSTPARTY: str = "FIRSTPARTY" +LOCALFOLDER: str = "LOCALFOLDER" +DEFAULT: Tuple[str, ...] = (FUTURE, STDLIB, THIRDPARTY, FIRSTPARTY, LOCALFOLDER) diff --git a/WebCrawler/venv/Lib/site-packages/isort/settings.py b/WebCrawler/venv/Lib/site-packages/isort/settings.py new file mode 100644 index 0000000..96fc941 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/isort/settings.py @@ -0,0 +1,739 @@ +"""isort/settings.py. + +Defines how the default settings for isort should be loaded +""" +import configparser +import fnmatch +import os +import posixpath +import re +import stat +import subprocess # nosec: Needed for gitignore support. +import sys +from functools import lru_cache +from pathlib import Path +from typing import Any, Callable, Dict, FrozenSet, Iterable, List, Optional, Pattern, Set, Tuple +from warnings import warn + +from . import stdlibs +from ._future import dataclass, field +from ._vendored import toml +from .exceptions import ( + FormattingPluginDoesNotExist, + InvalidSettingsPath, + ProfileDoesNotExist, + UnsupportedSettings, +) +from .profiles import profiles +from .sections import DEFAULT as SECTION_DEFAULTS +from .sections import FIRSTPARTY, FUTURE, LOCALFOLDER, STDLIB, THIRDPARTY +from .wrap_modes import WrapModes +from .wrap_modes import from_string as wrap_mode_from_string + +_SHEBANG_RE = re.compile(br"^#!.*\bpython[23w]?\b") +SUPPORTED_EXTENSIONS = frozenset({"py", "pyi", "pyx", "pxd"}) +BLOCKED_EXTENSIONS = frozenset({"pex"}) +FILE_SKIP_COMMENTS: Tuple[str, ...] = ( + "isort:" + "skip_file", + "isort: " + "skip_file", +) # Concatenated to avoid this file being skipped +MAX_CONFIG_SEARCH_DEPTH: int = 25 # The number of parent directories to for a config file within +STOP_CONFIG_SEARCH_ON_DIRS: Tuple[str, ...] = (".git", ".hg") +VALID_PY_TARGETS: Tuple[str, ...] = tuple( + target.replace("py", "") for target in dir(stdlibs) if not target.startswith("_") +) +CONFIG_SOURCES: Tuple[str, ...] = ( + ".isort.cfg", + "pyproject.toml", + "setup.cfg", + "tox.ini", + ".editorconfig", +) +DEFAULT_SKIP: FrozenSet[str] = frozenset( + { + ".venv", + "venv", + ".tox", + ".eggs", + ".git", + ".hg", + ".mypy_cache", + ".nox", + ".svn", + ".bzr", + "_build", + "buck-out", + "build", + "dist", + ".pants.d", + ".direnv", + "node_modules", + } +) + +CONFIG_SECTIONS: Dict[str, Tuple[str, ...]] = { + ".isort.cfg": ("settings", "isort"), + "pyproject.toml": ("tool.isort",), + "setup.cfg": ("isort", "tool:isort"), + "tox.ini": ("isort", "tool:isort"), + ".editorconfig": ("*", "*.py", "**.py", "*.{py}"), +} +FALLBACK_CONFIG_SECTIONS: Tuple[str, ...] = ("isort", "tool:isort", "tool.isort") + +IMPORT_HEADING_PREFIX = "import_heading_" +KNOWN_PREFIX = "known_" +KNOWN_SECTION_MAPPING: Dict[str, str] = { + STDLIB: "STANDARD_LIBRARY", + FUTURE: "FUTURE_LIBRARY", + FIRSTPARTY: "FIRST_PARTY", + THIRDPARTY: "THIRD_PARTY", + LOCALFOLDER: "LOCAL_FOLDER", +} + +RUNTIME_SOURCE = "runtime" + +DEPRECATED_SETTINGS = ("not_skip", "keep_direct_and_as_imports") + +_STR_BOOLEAN_MAPPING = { + "y": True, + "yes": True, + "t": True, + "on": True, + "1": True, + "true": True, + "n": False, + "no": False, + "f": False, + "off": False, + "0": False, + "false": False, +} + + +@dataclass(frozen=True) +class _Config: + """Defines the data schema and defaults used for isort configuration. + + NOTE: known lists, such as known_standard_library, are intentionally not complete as they are + dynamically determined later on. + """ + + py_version: str = "3" + force_to_top: FrozenSet[str] = frozenset() + skip: FrozenSet[str] = DEFAULT_SKIP + skip_glob: FrozenSet[str] = frozenset() + skip_gitignore: bool = False + line_length: int = 79 + wrap_length: int = 0 + line_ending: str = "" + sections: Tuple[str, ...] = SECTION_DEFAULTS + no_sections: bool = False + known_future_library: FrozenSet[str] = frozenset(("__future__",)) + known_third_party: FrozenSet[str] = frozenset() + known_first_party: FrozenSet[str] = frozenset() + known_local_folder: FrozenSet[str] = frozenset() + known_standard_library: FrozenSet[str] = frozenset() + extra_standard_library: FrozenSet[str] = frozenset() + known_other: Dict[str, FrozenSet[str]] = field(default_factory=dict) + multi_line_output: WrapModes = WrapModes.GRID # type: ignore + forced_separate: Tuple[str, ...] = () + indent: str = " " * 4 + comment_prefix: str = " #" + length_sort: bool = False + length_sort_straight: bool = False + length_sort_sections: FrozenSet[str] = frozenset() + add_imports: FrozenSet[str] = frozenset() + remove_imports: FrozenSet[str] = frozenset() + append_only: bool = False + reverse_relative: bool = False + force_single_line: bool = False + single_line_exclusions: Tuple[str, ...] = () + default_section: str = THIRDPARTY + import_headings: Dict[str, str] = field(default_factory=dict) + balanced_wrapping: bool = False + use_parentheses: bool = False + order_by_type: bool = True + atomic: bool = False + lines_after_imports: int = -1 + lines_between_sections: int = 1 + lines_between_types: int = 0 + combine_as_imports: bool = False + combine_star: bool = False + include_trailing_comma: bool = False + from_first: bool = False + verbose: bool = False + quiet: bool = False + force_adds: bool = False + force_alphabetical_sort_within_sections: bool = False + force_alphabetical_sort: bool = False + force_grid_wrap: int = 0 + force_sort_within_sections: bool = False + lexicographical: bool = False + group_by_package: bool = False + ignore_whitespace: bool = False + no_lines_before: FrozenSet[str] = frozenset() + no_inline_sort: bool = False + ignore_comments: bool = False + case_sensitive: bool = False + sources: Tuple[Dict[str, Any], ...] = () + virtual_env: str = "" + conda_env: str = "" + ensure_newline_before_comments: bool = False + directory: str = "" + profile: str = "" + honor_noqa: bool = False + src_paths: Tuple[Path, ...] = () + old_finders: bool = False + remove_redundant_aliases: bool = False + float_to_top: bool = False + filter_files: bool = False + formatter: str = "" + formatting_function: Optional[Callable[[str, str, object], str]] = None + color_output: bool = False + treat_comments_as_code: FrozenSet[str] = frozenset() + treat_all_comments_as_code: bool = False + supported_extensions: FrozenSet[str] = SUPPORTED_EXTENSIONS + blocked_extensions: FrozenSet[str] = BLOCKED_EXTENSIONS + constants: FrozenSet[str] = frozenset() + classes: FrozenSet[str] = frozenset() + variables: FrozenSet[str] = frozenset() + dedup_headings: bool = False + only_sections: bool = False + only_modified: bool = False + auto_identify_namespace_packages: bool = True + namespace_packages: FrozenSet[str] = frozenset() + + def __post_init__(self): + py_version = self.py_version + if py_version == "auto": # pragma: no cover + if sys.version_info.major == 2 and sys.version_info.minor <= 6: + py_version = "2" + elif sys.version_info.major == 3 and ( + sys.version_info.minor <= 5 or sys.version_info.minor >= 9 + ): + py_version = "3" + else: + py_version = f"{sys.version_info.major}{sys.version_info.minor}" + + if py_version not in VALID_PY_TARGETS: + raise ValueError( + f"The python version {py_version} is not supported. " + "You can set a python version with the -py or --python-version flag. " + f"The following versions are supported: {VALID_PY_TARGETS}" + ) + + if py_version != "all": + object.__setattr__(self, "py_version", f"py{py_version}") + + if not self.known_standard_library: + object.__setattr__( + self, "known_standard_library", frozenset(getattr(stdlibs, self.py_version).stdlib) + ) + + if self.force_alphabetical_sort: + object.__setattr__(self, "force_alphabetical_sort_within_sections", True) + object.__setattr__(self, "no_sections", True) + object.__setattr__(self, "lines_between_types", 1) + object.__setattr__(self, "from_first", True) + if self.wrap_length > self.line_length: + raise ValueError( + "wrap_length must be set lower than or equal to line_length: " + f"{self.wrap_length} > {self.line_length}." + ) + + def __hash__(self): + return id(self) + + +_DEFAULT_SETTINGS = {**vars(_Config()), "source": "defaults"} + + +class Config(_Config): + def __init__( + self, + settings_file: str = "", + settings_path: str = "", + config: Optional[_Config] = None, + **config_overrides, + ): + self._known_patterns: Optional[List[Tuple[Pattern[str], str]]] = None + self._section_comments: Optional[Tuple[str, ...]] = None + + if config: + config_vars = vars(config).copy() + config_vars.update(config_overrides) + config_vars["py_version"] = config_vars["py_version"].replace("py", "") + config_vars.pop("_known_patterns") + config_vars.pop("_section_comments") + super().__init__(**config_vars) # type: ignore + return + + # We can't use self.quiet to conditionally show warnings before super.__init__() is called + # at the end of this method. _Config is also frozen so setting self.quiet isn't possible. + # Therefore we extract quiet early here in a variable and use that in warning conditions. + quiet = config_overrides.get("quiet", False) + + sources: List[Dict[str, Any]] = [_DEFAULT_SETTINGS] + + config_settings: Dict[str, Any] + project_root: str + if settings_file: + config_settings = _get_config_data( + settings_file, + CONFIG_SECTIONS.get(os.path.basename(settings_file), FALLBACK_CONFIG_SECTIONS), + ) + project_root = os.path.dirname(settings_file) + if not config_settings and not quiet: + warn( + f"A custom settings file was specified: {settings_file} but no configuration " + "was found inside. This can happen when [settings] is used as the config " + "header instead of [isort]. " + "See: https://pycqa.github.io/isort/docs/configuration/config_files" + "/#custom_config_files for more information." + ) + elif settings_path: + if not os.path.exists(settings_path): + raise InvalidSettingsPath(settings_path) + + settings_path = os.path.abspath(settings_path) + project_root, config_settings = _find_config(settings_path) + else: + config_settings = {} + project_root = os.getcwd() + + profile_name = config_overrides.get("profile", config_settings.get("profile", "")) + profile: Dict[str, Any] = {} + if profile_name: + if profile_name not in profiles: + import pkg_resources + + for plugin in pkg_resources.iter_entry_points("isort.profiles"): + profiles.setdefault(plugin.name, plugin.load()) + + if profile_name not in profiles: + raise ProfileDoesNotExist(profile_name) + + profile = profiles[profile_name].copy() + profile["source"] = f"{profile_name} profile" + sources.append(profile) + + if config_settings: + sources.append(config_settings) + if config_overrides: + config_overrides["source"] = RUNTIME_SOURCE + sources.append(config_overrides) + + combined_config = {**profile, **config_settings, **config_overrides} + if "indent" in combined_config: + indent = str(combined_config["indent"]) + if indent.isdigit(): + indent = " " * int(indent) + else: + indent = indent.strip("'").strip('"') + if indent.lower() == "tab": + indent = "\t" + combined_config["indent"] = indent + + known_other = {} + import_headings = {} + for key, value in tuple(combined_config.items()): + # Collect all known sections beyond those that have direct entries + if key.startswith(KNOWN_PREFIX) and key not in ( + "known_standard_library", + "known_future_library", + "known_third_party", + "known_first_party", + "known_local_folder", + ): + import_heading = key[len(KNOWN_PREFIX) :].lower() + maps_to_section = import_heading.upper() + combined_config.pop(key) + if maps_to_section in KNOWN_SECTION_MAPPING: + section_name = f"known_{KNOWN_SECTION_MAPPING[maps_to_section].lower()}" + if section_name in combined_config and not quiet: + warn( + f"Can't set both {key} and {section_name} in the same config file.\n" + f"Default to {section_name} if unsure." + "\n\n" + "See: https://pycqa.github.io/isort/" + "#custom-sections-and-ordering." + ) + else: + combined_config[section_name] = frozenset(value) + else: + known_other[import_heading] = frozenset(value) + if maps_to_section not in combined_config.get("sections", ()) and not quiet: + warn( + f"`{key}` setting is defined, but {maps_to_section} is not" + " included in `sections` config option:" + f" {combined_config.get('sections', SECTION_DEFAULTS)}.\n\n" + "See: https://pycqa.github.io/isort/" + "#custom-sections-and-ordering." + ) + if key.startswith(IMPORT_HEADING_PREFIX): + import_headings[key[len(IMPORT_HEADING_PREFIX) :].lower()] = str(value) + + # Coerce all provided config values into their correct type + default_value = _DEFAULT_SETTINGS.get(key, None) + if default_value is None: + continue + + combined_config[key] = type(default_value)(value) + + for section in combined_config.get("sections", ()): + if section in SECTION_DEFAULTS: + continue + elif not section.lower() in known_other: + config_keys = ", ".join(known_other.keys()) + warn( + f"`sections` setting includes {section}, but no known_{section.lower()} " + "is defined. " + f"The following known_SECTION config options are defined: {config_keys}." + ) + + if "directory" not in combined_config: + combined_config["directory"] = ( + os.path.dirname(config_settings["source"]) + if config_settings.get("source", None) + else os.getcwd() + ) + + path_root = Path(combined_config.get("directory", project_root)).resolve() + path_root = path_root if path_root.is_dir() else path_root.parent + if "src_paths" not in combined_config: + combined_config["src_paths"] = (path_root / "src", path_root) + else: + src_paths: List[Path] = [] + for src_path in combined_config.get("src_paths", ()): + full_path = path_root / src_path + if full_path not in src_paths: + src_paths.append(full_path) + + combined_config["src_paths"] = tuple(src_paths) + + if "formatter" in combined_config: + import pkg_resources + + for plugin in pkg_resources.iter_entry_points("isort.formatters"): + if plugin.name == combined_config["formatter"]: + combined_config["formatting_function"] = plugin.load() + break + else: + raise FormattingPluginDoesNotExist(combined_config["formatter"]) + + # Remove any config values that are used for creating config object but + # aren't defined in dataclass + combined_config.pop("source", None) + combined_config.pop("sources", None) + combined_config.pop("runtime_src_paths", None) + + deprecated_options_used = [ + option for option in combined_config if option in DEPRECATED_SETTINGS + ] + if deprecated_options_used: + for deprecated_option in deprecated_options_used: + combined_config.pop(deprecated_option) + if not quiet: + warn( + "W0503: Deprecated config options were used: " + f"{', '.join(deprecated_options_used)}." + "Please see the 5.0.0 upgrade guide: bit.ly/isortv5." + ) + + if known_other: + combined_config["known_other"] = known_other + if import_headings: + for import_heading_key in import_headings: + combined_config.pop(f"{IMPORT_HEADING_PREFIX}{import_heading_key}") + combined_config["import_headings"] = import_headings + + unsupported_config_errors = {} + for option in set(combined_config.keys()).difference( + getattr(_Config, "__dataclass_fields__", {}).keys() + ): + for source in reversed(sources): + if option in source: + unsupported_config_errors[option] = { + "value": source[option], + "source": source["source"], + } + if unsupported_config_errors: + raise UnsupportedSettings(unsupported_config_errors) + + super().__init__(sources=tuple(sources), **combined_config) # type: ignore + + def is_supported_filetype(self, file_name: str): + _root, ext = os.path.splitext(file_name) + ext = ext.lstrip(".") + if ext in self.supported_extensions: + return True + elif ext in self.blocked_extensions: + return False + + # Skip editor backup files. + if file_name.endswith("~"): + return False + + try: + if stat.S_ISFIFO(os.stat(file_name).st_mode): + return False + except OSError: + pass + + try: + with open(file_name, "rb") as fp: + line = fp.readline(100) + except OSError: + return False + else: + return bool(_SHEBANG_RE.match(line)) + + def is_skipped(self, file_path: Path) -> bool: + """Returns True if the file and/or folder should be skipped based on current settings.""" + if self.directory and Path(self.directory) in file_path.resolve().parents: + file_name = os.path.relpath(file_path.resolve(), self.directory) + else: + file_name = str(file_path) + + os_path = str(file_path) + + if self.skip_gitignore: + if file_path.name == ".git": # pragma: no cover + return True + + result = subprocess.run( # nosec + ["git", "-C", str(file_path.parent), "check-ignore", "--quiet", os_path] + ) + if result.returncode == 0: + return True + + normalized_path = os_path.replace("\\", "/") + if normalized_path[1:2] == ":": + normalized_path = normalized_path[2:] + + for skip_path in self.skip: + if posixpath.abspath(normalized_path) == posixpath.abspath( + skip_path.replace("\\", "/") + ): + return True + + position = os.path.split(file_name) + while position[1]: + if position[1] in self.skip: + return True + position = os.path.split(position[0]) + + for glob in self.skip_glob: + if fnmatch.fnmatch(file_name, glob) or fnmatch.fnmatch("/" + file_name, glob): + return True + + if not (os.path.isfile(os_path) or os.path.isdir(os_path) or os.path.islink(os_path)): + return True + + return False + + @property + def known_patterns(self): + if self._known_patterns is not None: + return self._known_patterns + + self._known_patterns = [] + pattern_sections = [STDLIB] + [section for section in self.sections if section != STDLIB] + for placement in reversed(pattern_sections): + known_placement = KNOWN_SECTION_MAPPING.get(placement, placement).lower() + config_key = f"{KNOWN_PREFIX}{known_placement}" + known_modules = getattr(self, config_key, self.known_other.get(known_placement, ())) + extra_modules = getattr(self, f"extra_{known_placement}", ()) + all_modules = set(extra_modules).union(known_modules) + known_patterns = [ + pattern + for known_pattern in all_modules + for pattern in self._parse_known_pattern(known_pattern) + ] + for known_pattern in known_patterns: + regexp = "^" + known_pattern.replace("*", ".*").replace("?", ".?") + "$" + self._known_patterns.append((re.compile(regexp), placement)) + + return self._known_patterns + + @property + def section_comments(self) -> Tuple[str, ...]: + if self._section_comments is not None: + return self._section_comments + + self._section_comments = tuple(f"# {heading}" for heading in self.import_headings.values()) + return self._section_comments + + def _parse_known_pattern(self, pattern: str) -> List[str]: + """Expand pattern if identified as a directory and return found sub packages""" + if pattern.endswith(os.path.sep): + patterns = [ + filename + for filename in os.listdir(os.path.join(self.directory, pattern)) + if os.path.isdir(os.path.join(self.directory, pattern, filename)) + ] + else: + patterns = [pattern] + + return patterns + + +def _get_str_to_type_converter(setting_name: str) -> Callable[[str], Any]: + type_converter: Callable[[str], Any] = type(_DEFAULT_SETTINGS.get(setting_name, "")) + if type_converter == WrapModes: + type_converter = wrap_mode_from_string + return type_converter + + +def _as_list(value: str) -> List[str]: + if isinstance(value, list): + return [item.strip() for item in value] + filtered = [item.strip() for item in value.replace("\n", ",").split(",") if item.strip()] + return filtered + + +def _abspaths(cwd: str, values: Iterable[str]) -> Set[str]: + paths = { + os.path.join(cwd, value) + if not value.startswith(os.path.sep) and value.endswith(os.path.sep) + else value + for value in values + } + return paths + + +@lru_cache() +def _find_config(path: str) -> Tuple[str, Dict[str, Any]]: + current_directory = path + tries = 0 + while current_directory and tries < MAX_CONFIG_SEARCH_DEPTH: + for config_file_name in CONFIG_SOURCES: + potential_config_file = os.path.join(current_directory, config_file_name) + if os.path.isfile(potential_config_file): + config_data: Dict[str, Any] + try: + config_data = _get_config_data( + potential_config_file, CONFIG_SECTIONS[config_file_name] + ) + except Exception: + warn(f"Failed to pull configuration information from {potential_config_file}") + config_data = {} + if config_data: + return (current_directory, config_data) + + for stop_dir in STOP_CONFIG_SEARCH_ON_DIRS: + if os.path.isdir(os.path.join(current_directory, stop_dir)): + return (current_directory, {}) + + new_directory = os.path.split(current_directory)[0] + if new_directory == current_directory: + break + + current_directory = new_directory + tries += 1 + + return (path, {}) + + +@lru_cache() +def _get_config_data(file_path: str, sections: Tuple[str]) -> Dict[str, Any]: + settings: Dict[str, Any] = {} + + with open(file_path) as config_file: + if file_path.endswith(".toml"): + config = toml.load(config_file) + for section in sections: + config_section = config + for key in section.split("."): + config_section = config_section.get(key, {}) + settings.update(config_section) + else: + if file_path.endswith(".editorconfig"): + line = "\n" + last_position = config_file.tell() + while line: + line = config_file.readline() + if "[" in line: + config_file.seek(last_position) + break + last_position = config_file.tell() + + config = configparser.ConfigParser(strict=False) + config.read_file(config_file) + for section in sections: + if section.startswith("*.{") and section.endswith("}"): + extension = section[len("*.{") : -1] + for config_key in config.keys(): + if config_key.startswith("*.{") and config_key.endswith("}"): + if extension in map( + lambda text: text.strip(), config_key[len("*.{") : -1].split(",") + ): + settings.update(config.items(config_key)) + + elif config.has_section(section): + settings.update(config.items(section)) + + if settings: + settings["source"] = file_path + + if file_path.endswith(".editorconfig"): + indent_style = settings.pop("indent_style", "").strip() + indent_size = settings.pop("indent_size", "").strip() + if indent_size == "tab": + indent_size = settings.pop("tab_width", "").strip() + + if indent_style == "space": + settings["indent"] = " " * (indent_size and int(indent_size) or 4) + + elif indent_style == "tab": + settings["indent"] = "\t" * (indent_size and int(indent_size) or 1) + + max_line_length = settings.pop("max_line_length", "").strip() + if max_line_length and (max_line_length == "off" or max_line_length.isdigit()): + settings["line_length"] = ( + float("inf") if max_line_length == "off" else int(max_line_length) + ) + settings = { + key: value + for key, value in settings.items() + if key in _DEFAULT_SETTINGS.keys() or key.startswith(KNOWN_PREFIX) + } + + for key, value in settings.items(): + existing_value_type = _get_str_to_type_converter(key) + if existing_value_type == tuple: + settings[key] = tuple(_as_list(value)) + elif existing_value_type == frozenset: + settings[key] = frozenset(_as_list(settings.get(key))) # type: ignore + elif existing_value_type == bool: + # Only some configuration formats support native boolean values. + if not isinstance(value, bool): + value = _as_bool(value) + settings[key] = value + elif key.startswith(KNOWN_PREFIX): + settings[key] = _abspaths(os.path.dirname(file_path), _as_list(value)) + elif key == "force_grid_wrap": + try: + result = existing_value_type(value) + except ValueError: # backwards compatibility for true / false force grid wrap + result = 0 if value.lower().strip() == "false" else 2 + settings[key] = result + elif key == "comment_prefix": + settings[key] = str(value).strip("'").strip('"') + else: + settings[key] = existing_value_type(value) + + return settings + + +def _as_bool(value: str) -> bool: + """Given a string value that represents True or False, returns the Boolean equivalent. + Heavily inspired from distutils strtobool. + """ + try: + return _STR_BOOLEAN_MAPPING[value.lower()] + except KeyError: + raise ValueError(f"invalid truth value {value}") + + +DEFAULT_CONFIG = Config() diff --git a/WebCrawler/venv/Lib/site-packages/isort/setuptools_commands.py b/WebCrawler/venv/Lib/site-packages/isort/setuptools_commands.py new file mode 100644 index 0000000..96e41dd --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/isort/setuptools_commands.py @@ -0,0 +1,61 @@ +import glob +import os +import sys +from typing import Any, Dict, Iterator, List +from warnings import warn + +import setuptools + +from . import api +from .settings import DEFAULT_CONFIG + + +class ISortCommand(setuptools.Command): + """The :class:`ISortCommand` class is used by setuptools to perform + imports checks on registered modules. + """ + + description = "Run isort on modules registered in setuptools" + user_options: List[Any] = [] + + def initialize_options(self) -> None: + default_settings = vars(DEFAULT_CONFIG).copy() + for key, value in default_settings.items(): + setattr(self, key, value) + + def finalize_options(self) -> None: + "Get options from config files." + self.arguments: Dict[str, Any] = {} # skipcq: PYL-W0201 + self.arguments["settings_path"] = os.getcwd() + + def distribution_files(self) -> Iterator[str]: + """Find distribution packages.""" + # This is verbatim from flake8 + if self.distribution.packages: # pragma: no cover + package_dirs = self.distribution.package_dir or {} + for package in self.distribution.packages: + pkg_dir = package + if package in package_dirs: + pkg_dir = package_dirs[package] + elif "" in package_dirs: # pragma: no cover + pkg_dir = package_dirs[""] + os.path.sep + pkg_dir + yield pkg_dir.replace(".", os.path.sep) + + if self.distribution.py_modules: + for filename in self.distribution.py_modules: + yield "%s.py" % filename + # Don't miss the setup.py file itself + yield "setup.py" + + def run(self) -> None: + arguments = self.arguments + wrong_sorted_files = False + for path in self.distribution_files(): + for python_file in glob.iglob(os.path.join(path, "*.py")): + try: + if not api.check_file(python_file, **arguments): + wrong_sorted_files = True # pragma: no cover + except OSError as error: # pragma: no cover + warn(f"Unable to parse file {python_file} due to {error}") + if wrong_sorted_files: + sys.exit(1) # pragma: no cover diff --git a/WebCrawler/venv/Lib/site-packages/isort/sorting.py b/WebCrawler/venv/Lib/site-packages/isort/sorting.py new file mode 100644 index 0000000..cab7701 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/isort/sorting.py @@ -0,0 +1,102 @@ +import re +from typing import Any, Callable, Iterable, List, Optional + +from .settings import Config + +_import_line_intro_re = re.compile("^(?:from|import) ") +_import_line_midline_import_re = re.compile(" import ") + + +def module_key( + module_name: str, + config: Config, + sub_imports: bool = False, + ignore_case: bool = False, + section_name: Optional[Any] = None, + straight_import: Optional[bool] = False, +) -> str: + match = re.match(r"^(\.+)\s*(.*)", module_name) + if match: + sep = " " if config.reverse_relative else "_" + module_name = sep.join(match.groups()) + + prefix = "" + if ignore_case: + module_name = str(module_name).lower() + else: + module_name = str(module_name) + + if sub_imports and config.order_by_type: + if module_name in config.constants: + prefix = "A" + elif module_name in config.classes: + prefix = "B" + elif module_name in config.variables: + prefix = "C" + elif module_name.isupper() and len(module_name) > 1: # see issue #376 + prefix = "A" + elif module_name in config.classes or module_name[0:1].isupper(): + prefix = "B" + else: + prefix = "C" + if not config.case_sensitive: + module_name = module_name.lower() + + length_sort = ( + config.length_sort + or (config.length_sort_straight and straight_import) + or str(section_name).lower() in config.length_sort_sections + ) + _length_sort_maybe = length_sort and (str(len(module_name)) + ":" + module_name) or module_name + return f"{module_name in config.force_to_top and 'A' or 'B'}{prefix}{_length_sort_maybe}" + + +def section_key( + line: str, + order_by_type: bool, + force_to_top: List[str], + lexicographical: bool = False, + length_sort: bool = False, + reverse_relative: bool = False, + group_by_package: bool = False, +) -> str: + section = "B" + + if reverse_relative and line.startswith("from ."): + match = re.match(r"^from (\.+)\s*(.*)", line) + if match: # pragma: no cover - regex always matches if line starts with "from ." + line = f"from {' '.join(match.groups())}" + if group_by_package and line.strip().startswith("from"): + line = line.split(" import", 1)[0] + + if lexicographical: + line = _import_line_intro_re.sub("", _import_line_midline_import_re.sub(".", line)) + else: + line = re.sub("^from ", "", line) + line = re.sub("^import ", "", line) + if line.split(" ")[0] in force_to_top: + section = "A" + if not order_by_type: + line = line.lower() + + return f"{section}{len(line) if length_sort else ''}{line}" + + +def naturally(to_sort: Iterable[str], key: Optional[Callable[[str], Any]] = None) -> List[str]: + """Returns a naturally sorted list""" + if key is None: + key_callback = _natural_keys + else: + + def key_callback(text: str) -> List[Any]: + return _natural_keys(key(text)) # type: ignore + + return sorted(to_sort, key=key_callback) + + +def _atoi(text: str) -> Any: + return int(text) if text.isdigit() else text + + +def _natural_keys(text: str) -> List[Any]: + return [_atoi(c) for c in re.split(r"(\d+)", text)] diff --git a/WebCrawler/venv/Lib/site-packages/isort/stdlibs/__init__.py b/WebCrawler/venv/Lib/site-packages/isort/stdlibs/__init__.py new file mode 100644 index 0000000..9021bc4 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/isort/stdlibs/__init__.py @@ -0,0 +1 @@ +from . import all, py2, py3, py27, py35, py36, py37, py38, py39 diff --git a/WebCrawler/venv/Lib/site-packages/isort/stdlibs/__pycache__/__init__.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/isort/stdlibs/__pycache__/__init__.cpython-39.pyc new file mode 100644 index 0000000..1818ce7 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/isort/stdlibs/__pycache__/__init__.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/isort/stdlibs/__pycache__/all.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/isort/stdlibs/__pycache__/all.cpython-39.pyc new file mode 100644 index 0000000..a3abb53 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/isort/stdlibs/__pycache__/all.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/isort/stdlibs/__pycache__/py2.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/isort/stdlibs/__pycache__/py2.cpython-39.pyc new file mode 100644 index 0000000..85d8ff7 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/isort/stdlibs/__pycache__/py2.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/isort/stdlibs/__pycache__/py27.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/isort/stdlibs/__pycache__/py27.cpython-39.pyc new file mode 100644 index 0000000..2999db4 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/isort/stdlibs/__pycache__/py27.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/isort/stdlibs/__pycache__/py3.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/isort/stdlibs/__pycache__/py3.cpython-39.pyc new file mode 100644 index 0000000..9f04ffb Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/isort/stdlibs/__pycache__/py3.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/isort/stdlibs/__pycache__/py35.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/isort/stdlibs/__pycache__/py35.cpython-39.pyc new file mode 100644 index 0000000..beba141 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/isort/stdlibs/__pycache__/py35.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/isort/stdlibs/__pycache__/py36.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/isort/stdlibs/__pycache__/py36.cpython-39.pyc new file mode 100644 index 0000000..9bf6684 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/isort/stdlibs/__pycache__/py36.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/isort/stdlibs/__pycache__/py37.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/isort/stdlibs/__pycache__/py37.cpython-39.pyc new file mode 100644 index 0000000..83e1df8 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/isort/stdlibs/__pycache__/py37.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/isort/stdlibs/__pycache__/py38.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/isort/stdlibs/__pycache__/py38.cpython-39.pyc new file mode 100644 index 0000000..aed964e Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/isort/stdlibs/__pycache__/py38.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/isort/stdlibs/__pycache__/py39.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/isort/stdlibs/__pycache__/py39.cpython-39.pyc new file mode 100644 index 0000000..2485eeb Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/isort/stdlibs/__pycache__/py39.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/isort/stdlibs/all.py b/WebCrawler/venv/Lib/site-packages/isort/stdlibs/all.py new file mode 100644 index 0000000..08a365e --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/isort/stdlibs/all.py @@ -0,0 +1,3 @@ +from . import py2, py3 + +stdlib = py2.stdlib | py3.stdlib diff --git a/WebCrawler/venv/Lib/site-packages/isort/stdlibs/py2.py b/WebCrawler/venv/Lib/site-packages/isort/stdlibs/py2.py new file mode 100644 index 0000000..74af019 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/isort/stdlibs/py2.py @@ -0,0 +1,3 @@ +from . import py27 + +stdlib = py27.stdlib diff --git a/WebCrawler/venv/Lib/site-packages/isort/stdlibs/py27.py b/WebCrawler/venv/Lib/site-packages/isort/stdlibs/py27.py new file mode 100644 index 0000000..87aa67f --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/isort/stdlibs/py27.py @@ -0,0 +1,300 @@ +""" +File contains the standard library of Python 2.7. + +DO NOT EDIT. If the standard library changes, a new list should be created +using the mkstdlibs.py script. +""" + +stdlib = { + "AL", + "BaseHTTPServer", + "Bastion", + "CGIHTTPServer", + "Carbon", + "ColorPicker", + "ConfigParser", + "Cookie", + "DEVICE", + "DocXMLRPCServer", + "EasyDialogs", + "FL", + "FrameWork", + "GL", + "HTMLParser", + "MacOS", + "MimeWriter", + "MiniAEFrame", + "Nav", + "PixMapWrapper", + "Queue", + "SUNAUDIODEV", + "ScrolledText", + "SimpleHTTPServer", + "SimpleXMLRPCServer", + "SocketServer", + "StringIO", + "Tix", + "Tkinter", + "UserDict", + "UserList", + "UserString", + "W", + "__builtin__", + "_winreg", + "abc", + "aepack", + "aetools", + "aetypes", + "aifc", + "al", + "anydbm", + "applesingle", + "argparse", + "array", + "ast", + "asynchat", + "asyncore", + "atexit", + "audioop", + "autoGIL", + "base64", + "bdb", + "binascii", + "binhex", + "bisect", + "bsddb", + "buildtools", + "bz2", + "cPickle", + "cProfile", + "cStringIO", + "calendar", + "cd", + "cfmfile", + "cgi", + "cgitb", + "chunk", + "cmath", + "cmd", + "code", + "codecs", + "codeop", + "collections", + "colorsys", + "commands", + "compileall", + "compiler", + "contextlib", + "cookielib", + "copy", + "copy_reg", + "crypt", + "csv", + "ctypes", + "curses", + "datetime", + "dbhash", + "dbm", + "decimal", + "difflib", + "dircache", + "dis", + "distutils", + "dl", + "doctest", + "dumbdbm", + "dummy_thread", + "dummy_threading", + "email", + "encodings", + "ensurepip", + "errno", + "exceptions", + "fcntl", + "filecmp", + "fileinput", + "findertools", + "fl", + "flp", + "fm", + "fnmatch", + "formatter", + "fpectl", + "fpformat", + "fractions", + "ftplib", + "functools", + "future_builtins", + "gc", + "gdbm", + "gensuitemodule", + "getopt", + "getpass", + "gettext", + "gl", + "glob", + "grp", + "gzip", + "hashlib", + "heapq", + "hmac", + "hotshot", + "htmlentitydefs", + "htmllib", + "httplib", + "ic", + "icopen", + "imageop", + "imaplib", + "imgfile", + "imghdr", + "imp", + "importlib", + "imputil", + "inspect", + "io", + "itertools", + "jpeg", + "json", + "keyword", + "lib2to3", + "linecache", + "locale", + "logging", + "macerrors", + "macostools", + "macpath", + "macresource", + "mailbox", + "mailcap", + "marshal", + "math", + "md5", + "mhlib", + "mimetools", + "mimetypes", + "mimify", + "mmap", + "modulefinder", + "msilib", + "msvcrt", + "multifile", + "multiprocessing", + "mutex", + "netrc", + "new", + "nis", + "nntplib", + "ntpath", + "numbers", + "operator", + "optparse", + "os", + "ossaudiodev", + "parser", + "pdb", + "pickle", + "pickletools", + "pipes", + "pkgutil", + "platform", + "plistlib", + "popen2", + "poplib", + "posix", + "posixfile", + "posixpath", + "pprint", + "profile", + "pstats", + "pty", + "pwd", + "py_compile", + "pyclbr", + "pydoc", + "quopri", + "random", + "re", + "readline", + "resource", + "rexec", + "rfc822", + "rlcompleter", + "robotparser", + "runpy", + "sched", + "select", + "sets", + "sgmllib", + "sha", + "shelve", + "shlex", + "shutil", + "signal", + "site", + "smtpd", + "smtplib", + "sndhdr", + "socket", + "spwd", + "sqlite3", + "sre", + "sre_compile", + "sre_constants", + "sre_parse", + "ssl", + "stat", + "statvfs", + "string", + "stringprep", + "struct", + "subprocess", + "sunau", + "sunaudiodev", + "symbol", + "symtable", + "sys", + "sysconfig", + "syslog", + "tabnanny", + "tarfile", + "telnetlib", + "tempfile", + "termios", + "test", + "textwrap", + "thread", + "threading", + "time", + "timeit", + "token", + "tokenize", + "trace", + "traceback", + "ttk", + "tty", + "turtle", + "types", + "unicodedata", + "unittest", + "urllib", + "urllib2", + "urlparse", + "user", + "uu", + "uuid", + "videoreader", + "warnings", + "wave", + "weakref", + "webbrowser", + "whichdb", + "winsound", + "wsgiref", + "xdrlib", + "xml", + "xmlrpclib", + "zipfile", + "zipimport", + "zlib", +} diff --git a/WebCrawler/venv/Lib/site-packages/isort/stdlibs/py3.py b/WebCrawler/venv/Lib/site-packages/isort/stdlibs/py3.py new file mode 100644 index 0000000..b7dbcc2 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/isort/stdlibs/py3.py @@ -0,0 +1,3 @@ +from . import py35, py36, py37, py38, py39 + +stdlib = py35.stdlib | py36.stdlib | py37.stdlib | py38.stdlib | py39.stdlib diff --git a/WebCrawler/venv/Lib/site-packages/isort/stdlibs/py35.py b/WebCrawler/venv/Lib/site-packages/isort/stdlibs/py35.py new file mode 100644 index 0000000..274d8a7 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/isort/stdlibs/py35.py @@ -0,0 +1,222 @@ +""" +File contains the standard library of Python 3.5. + +DO NOT EDIT. If the standard library changes, a new list should be created +using the mkstdlibs.py script. +""" + +stdlib = { + "_dummy_thread", + "_thread", + "abc", + "aifc", + "argparse", + "array", + "ast", + "asynchat", + "asyncio", + "asyncore", + "atexit", + "audioop", + "base64", + "bdb", + "binascii", + "binhex", + "bisect", + "builtins", + "bz2", + "cProfile", + "calendar", + "cgi", + "cgitb", + "chunk", + "cmath", + "cmd", + "code", + "codecs", + "codeop", + "collections", + "colorsys", + "compileall", + "concurrent", + "configparser", + "contextlib", + "copy", + "copyreg", + "crypt", + "csv", + "ctypes", + "curses", + "datetime", + "dbm", + "decimal", + "difflib", + "dis", + "distutils", + "doctest", + "dummy_threading", + "email", + "encodings", + "ensurepip", + "enum", + "errno", + "faulthandler", + "fcntl", + "filecmp", + "fileinput", + "fnmatch", + "formatter", + "fpectl", + "fractions", + "ftplib", + "functools", + "gc", + "getopt", + "getpass", + "gettext", + "glob", + "grp", + "gzip", + "hashlib", + "heapq", + "hmac", + "html", + "http", + "imaplib", + "imghdr", + "imp", + "importlib", + "inspect", + "io", + "ipaddress", + "itertools", + "json", + "keyword", + "lib2to3", + "linecache", + "locale", + "logging", + "lzma", + "macpath", + "mailbox", + "mailcap", + "marshal", + "math", + "mimetypes", + "mmap", + "modulefinder", + "msilib", + "msvcrt", + "multiprocessing", + "netrc", + "nis", + "nntplib", + "ntpath", + "numbers", + "operator", + "optparse", + "os", + "ossaudiodev", + "parser", + "pathlib", + "pdb", + "pickle", + "pickletools", + "pipes", + "pkgutil", + "platform", + "plistlib", + "poplib", + "posix", + "posixpath", + "pprint", + "profile", + "pstats", + "pty", + "pwd", + "py_compile", + "pyclbr", + "pydoc", + "queue", + "quopri", + "random", + "re", + "readline", + "reprlib", + "resource", + "rlcompleter", + "runpy", + "sched", + "select", + "selectors", + "shelve", + "shlex", + "shutil", + "signal", + "site", + "smtpd", + "smtplib", + "sndhdr", + "socket", + "socketserver", + "spwd", + "sqlite3", + "sre", + "sre_compile", + "sre_constants", + "sre_parse", + "ssl", + "stat", + "statistics", + "string", + "stringprep", + "struct", + "subprocess", + "sunau", + "symbol", + "symtable", + "sys", + "sysconfig", + "syslog", + "tabnanny", + "tarfile", + "telnetlib", + "tempfile", + "termios", + "test", + "textwrap", + "threading", + "time", + "timeit", + "tkinter", + "token", + "tokenize", + "trace", + "traceback", + "tracemalloc", + "tty", + "turtle", + "turtledemo", + "types", + "typing", + "unicodedata", + "unittest", + "urllib", + "uu", + "uuid", + "venv", + "warnings", + "wave", + "weakref", + "webbrowser", + "winreg", + "winsound", + "wsgiref", + "xdrlib", + "xml", + "xmlrpc", + "zipapp", + "zipfile", + "zipimport", + "zlib", +} diff --git a/WebCrawler/venv/Lib/site-packages/isort/stdlibs/py36.py b/WebCrawler/venv/Lib/site-packages/isort/stdlibs/py36.py new file mode 100644 index 0000000..8ae02a1 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/isort/stdlibs/py36.py @@ -0,0 +1,223 @@ +""" +File contains the standard library of Python 3.6. + +DO NOT EDIT. If the standard library changes, a new list should be created +using the mkstdlibs.py script. +""" + +stdlib = { + "_dummy_thread", + "_thread", + "abc", + "aifc", + "argparse", + "array", + "ast", + "asynchat", + "asyncio", + "asyncore", + "atexit", + "audioop", + "base64", + "bdb", + "binascii", + "binhex", + "bisect", + "builtins", + "bz2", + "cProfile", + "calendar", + "cgi", + "cgitb", + "chunk", + "cmath", + "cmd", + "code", + "codecs", + "codeop", + "collections", + "colorsys", + "compileall", + "concurrent", + "configparser", + "contextlib", + "copy", + "copyreg", + "crypt", + "csv", + "ctypes", + "curses", + "datetime", + "dbm", + "decimal", + "difflib", + "dis", + "distutils", + "doctest", + "dummy_threading", + "email", + "encodings", + "ensurepip", + "enum", + "errno", + "faulthandler", + "fcntl", + "filecmp", + "fileinput", + "fnmatch", + "formatter", + "fpectl", + "fractions", + "ftplib", + "functools", + "gc", + "getopt", + "getpass", + "gettext", + "glob", + "grp", + "gzip", + "hashlib", + "heapq", + "hmac", + "html", + "http", + "imaplib", + "imghdr", + "imp", + "importlib", + "inspect", + "io", + "ipaddress", + "itertools", + "json", + "keyword", + "lib2to3", + "linecache", + "locale", + "logging", + "lzma", + "macpath", + "mailbox", + "mailcap", + "marshal", + "math", + "mimetypes", + "mmap", + "modulefinder", + "msilib", + "msvcrt", + "multiprocessing", + "netrc", + "nis", + "nntplib", + "ntpath", + "numbers", + "operator", + "optparse", + "os", + "ossaudiodev", + "parser", + "pathlib", + "pdb", + "pickle", + "pickletools", + "pipes", + "pkgutil", + "platform", + "plistlib", + "poplib", + "posix", + "posixpath", + "pprint", + "profile", + "pstats", + "pty", + "pwd", + "py_compile", + "pyclbr", + "pydoc", + "queue", + "quopri", + "random", + "re", + "readline", + "reprlib", + "resource", + "rlcompleter", + "runpy", + "sched", + "secrets", + "select", + "selectors", + "shelve", + "shlex", + "shutil", + "signal", + "site", + "smtpd", + "smtplib", + "sndhdr", + "socket", + "socketserver", + "spwd", + "sqlite3", + "sre", + "sre_compile", + "sre_constants", + "sre_parse", + "ssl", + "stat", + "statistics", + "string", + "stringprep", + "struct", + "subprocess", + "sunau", + "symbol", + "symtable", + "sys", + "sysconfig", + "syslog", + "tabnanny", + "tarfile", + "telnetlib", + "tempfile", + "termios", + "test", + "textwrap", + "threading", + "time", + "timeit", + "tkinter", + "token", + "tokenize", + "trace", + "traceback", + "tracemalloc", + "tty", + "turtle", + "turtledemo", + "types", + "typing", + "unicodedata", + "unittest", + "urllib", + "uu", + "uuid", + "venv", + "warnings", + "wave", + "weakref", + "webbrowser", + "winreg", + "winsound", + "wsgiref", + "xdrlib", + "xml", + "xmlrpc", + "zipapp", + "zipfile", + "zipimport", + "zlib", +} diff --git a/WebCrawler/venv/Lib/site-packages/isort/stdlibs/py37.py b/WebCrawler/venv/Lib/site-packages/isort/stdlibs/py37.py new file mode 100644 index 0000000..0eb1dd6 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/isort/stdlibs/py37.py @@ -0,0 +1,224 @@ +""" +File contains the standard library of Python 3.7. + +DO NOT EDIT. If the standard library changes, a new list should be created +using the mkstdlibs.py script. +""" + +stdlib = { + "_dummy_thread", + "_thread", + "abc", + "aifc", + "argparse", + "array", + "ast", + "asynchat", + "asyncio", + "asyncore", + "atexit", + "audioop", + "base64", + "bdb", + "binascii", + "binhex", + "bisect", + "builtins", + "bz2", + "cProfile", + "calendar", + "cgi", + "cgitb", + "chunk", + "cmath", + "cmd", + "code", + "codecs", + "codeop", + "collections", + "colorsys", + "compileall", + "concurrent", + "configparser", + "contextlib", + "contextvars", + "copy", + "copyreg", + "crypt", + "csv", + "ctypes", + "curses", + "dataclasses", + "datetime", + "dbm", + "decimal", + "difflib", + "dis", + "distutils", + "doctest", + "dummy_threading", + "email", + "encodings", + "ensurepip", + "enum", + "errno", + "faulthandler", + "fcntl", + "filecmp", + "fileinput", + "fnmatch", + "formatter", + "fractions", + "ftplib", + "functools", + "gc", + "getopt", + "getpass", + "gettext", + "glob", + "grp", + "gzip", + "hashlib", + "heapq", + "hmac", + "html", + "http", + "imaplib", + "imghdr", + "imp", + "importlib", + "inspect", + "io", + "ipaddress", + "itertools", + "json", + "keyword", + "lib2to3", + "linecache", + "locale", + "logging", + "lzma", + "macpath", + "mailbox", + "mailcap", + "marshal", + "math", + "mimetypes", + "mmap", + "modulefinder", + "msilib", + "msvcrt", + "multiprocessing", + "netrc", + "nis", + "nntplib", + "ntpath", + "numbers", + "operator", + "optparse", + "os", + "ossaudiodev", + "parser", + "pathlib", + "pdb", + "pickle", + "pickletools", + "pipes", + "pkgutil", + "platform", + "plistlib", + "poplib", + "posix", + "posixpath", + "pprint", + "profile", + "pstats", + "pty", + "pwd", + "py_compile", + "pyclbr", + "pydoc", + "queue", + "quopri", + "random", + "re", + "readline", + "reprlib", + "resource", + "rlcompleter", + "runpy", + "sched", + "secrets", + "select", + "selectors", + "shelve", + "shlex", + "shutil", + "signal", + "site", + "smtpd", + "smtplib", + "sndhdr", + "socket", + "socketserver", + "spwd", + "sqlite3", + "sre", + "sre_compile", + "sre_constants", + "sre_parse", + "ssl", + "stat", + "statistics", + "string", + "stringprep", + "struct", + "subprocess", + "sunau", + "symbol", + "symtable", + "sys", + "sysconfig", + "syslog", + "tabnanny", + "tarfile", + "telnetlib", + "tempfile", + "termios", + "test", + "textwrap", + "threading", + "time", + "timeit", + "tkinter", + "token", + "tokenize", + "trace", + "traceback", + "tracemalloc", + "tty", + "turtle", + "turtledemo", + "types", + "typing", + "unicodedata", + "unittest", + "urllib", + "uu", + "uuid", + "venv", + "warnings", + "wave", + "weakref", + "webbrowser", + "winreg", + "winsound", + "wsgiref", + "xdrlib", + "xml", + "xmlrpc", + "zipapp", + "zipfile", + "zipimport", + "zlib", +} diff --git a/WebCrawler/venv/Lib/site-packages/isort/stdlibs/py38.py b/WebCrawler/venv/Lib/site-packages/isort/stdlibs/py38.py new file mode 100644 index 0000000..9bcea9a --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/isort/stdlibs/py38.py @@ -0,0 +1,223 @@ +""" +File contains the standard library of Python 3.8. + +DO NOT EDIT. If the standard library changes, a new list should be created +using the mkstdlibs.py script. +""" + +stdlib = { + "_dummy_thread", + "_thread", + "abc", + "aifc", + "argparse", + "array", + "ast", + "asynchat", + "asyncio", + "asyncore", + "atexit", + "audioop", + "base64", + "bdb", + "binascii", + "binhex", + "bisect", + "builtins", + "bz2", + "cProfile", + "calendar", + "cgi", + "cgitb", + "chunk", + "cmath", + "cmd", + "code", + "codecs", + "codeop", + "collections", + "colorsys", + "compileall", + "concurrent", + "configparser", + "contextlib", + "contextvars", + "copy", + "copyreg", + "crypt", + "csv", + "ctypes", + "curses", + "dataclasses", + "datetime", + "dbm", + "decimal", + "difflib", + "dis", + "distutils", + "doctest", + "dummy_threading", + "email", + "encodings", + "ensurepip", + "enum", + "errno", + "faulthandler", + "fcntl", + "filecmp", + "fileinput", + "fnmatch", + "formatter", + "fractions", + "ftplib", + "functools", + "gc", + "getopt", + "getpass", + "gettext", + "glob", + "grp", + "gzip", + "hashlib", + "heapq", + "hmac", + "html", + "http", + "imaplib", + "imghdr", + "imp", + "importlib", + "inspect", + "io", + "ipaddress", + "itertools", + "json", + "keyword", + "lib2to3", + "linecache", + "locale", + "logging", + "lzma", + "mailbox", + "mailcap", + "marshal", + "math", + "mimetypes", + "mmap", + "modulefinder", + "msilib", + "msvcrt", + "multiprocessing", + "netrc", + "nis", + "nntplib", + "ntpath", + "numbers", + "operator", + "optparse", + "os", + "ossaudiodev", + "parser", + "pathlib", + "pdb", + "pickle", + "pickletools", + "pipes", + "pkgutil", + "platform", + "plistlib", + "poplib", + "posix", + "posixpath", + "pprint", + "profile", + "pstats", + "pty", + "pwd", + "py_compile", + "pyclbr", + "pydoc", + "queue", + "quopri", + "random", + "re", + "readline", + "reprlib", + "resource", + "rlcompleter", + "runpy", + "sched", + "secrets", + "select", + "selectors", + "shelve", + "shlex", + "shutil", + "signal", + "site", + "smtpd", + "smtplib", + "sndhdr", + "socket", + "socketserver", + "spwd", + "sqlite3", + "sre", + "sre_compile", + "sre_constants", + "sre_parse", + "ssl", + "stat", + "statistics", + "string", + "stringprep", + "struct", + "subprocess", + "sunau", + "symbol", + "symtable", + "sys", + "sysconfig", + "syslog", + "tabnanny", + "tarfile", + "telnetlib", + "tempfile", + "termios", + "test", + "textwrap", + "threading", + "time", + "timeit", + "tkinter", + "token", + "tokenize", + "trace", + "traceback", + "tracemalloc", + "tty", + "turtle", + "turtledemo", + "types", + "typing", + "unicodedata", + "unittest", + "urllib", + "uu", + "uuid", + "venv", + "warnings", + "wave", + "weakref", + "webbrowser", + "winreg", + "winsound", + "wsgiref", + "xdrlib", + "xml", + "xmlrpc", + "zipapp", + "zipfile", + "zipimport", + "zlib", +} diff --git a/WebCrawler/venv/Lib/site-packages/isort/stdlibs/py39.py b/WebCrawler/venv/Lib/site-packages/isort/stdlibs/py39.py new file mode 100644 index 0000000..7bcb8f2 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/isort/stdlibs/py39.py @@ -0,0 +1,223 @@ +""" +File contains the standard library of Python 3.9. + +DO NOT EDIT. If the standard library changes, a new list should be created +using the mkstdlibs.py script. +""" + +stdlib = { + "_thread", + "abc", + "aifc", + "argparse", + "array", + "ast", + "asynchat", + "asyncio", + "asyncore", + "atexit", + "audioop", + "base64", + "bdb", + "binascii", + "binhex", + "bisect", + "builtins", + "bz2", + "cProfile", + "calendar", + "cgi", + "cgitb", + "chunk", + "cmath", + "cmd", + "code", + "codecs", + "codeop", + "collections", + "colorsys", + "compileall", + "concurrent", + "configparser", + "contextlib", + "contextvars", + "copy", + "copyreg", + "crypt", + "csv", + "ctypes", + "curses", + "dataclasses", + "datetime", + "dbm", + "decimal", + "difflib", + "dis", + "distutils", + "doctest", + "email", + "encodings", + "ensurepip", + "enum", + "errno", + "faulthandler", + "fcntl", + "filecmp", + "fileinput", + "fnmatch", + "formatter", + "fractions", + "ftplib", + "functools", + "gc", + "getopt", + "getpass", + "gettext", + "glob", + "graphlib", + "grp", + "gzip", + "hashlib", + "heapq", + "hmac", + "html", + "http", + "imaplib", + "imghdr", + "imp", + "importlib", + "inspect", + "io", + "ipaddress", + "itertools", + "json", + "keyword", + "lib2to3", + "linecache", + "locale", + "logging", + "lzma", + "mailbox", + "mailcap", + "marshal", + "math", + "mimetypes", + "mmap", + "modulefinder", + "msilib", + "msvcrt", + "multiprocessing", + "netrc", + "nis", + "nntplib", + "ntpath", + "numbers", + "operator", + "optparse", + "os", + "ossaudiodev", + "parser", + "pathlib", + "pdb", + "pickle", + "pickletools", + "pipes", + "pkgutil", + "platform", + "plistlib", + "poplib", + "posix", + "posixpath", + "pprint", + "profile", + "pstats", + "pty", + "pwd", + "py_compile", + "pyclbr", + "pydoc", + "queue", + "quopri", + "random", + "re", + "readline", + "reprlib", + "resource", + "rlcompleter", + "runpy", + "sched", + "secrets", + "select", + "selectors", + "shelve", + "shlex", + "shutil", + "signal", + "site", + "smtpd", + "smtplib", + "sndhdr", + "socket", + "socketserver", + "spwd", + "sqlite3", + "sre", + "sre_compile", + "sre_constants", + "sre_parse", + "ssl", + "stat", + "statistics", + "string", + "stringprep", + "struct", + "subprocess", + "sunau", + "symbol", + "symtable", + "sys", + "sysconfig", + "syslog", + "tabnanny", + "tarfile", + "telnetlib", + "tempfile", + "termios", + "test", + "textwrap", + "threading", + "time", + "timeit", + "tkinter", + "token", + "tokenize", + "trace", + "traceback", + "tracemalloc", + "tty", + "turtle", + "turtledemo", + "types", + "typing", + "unicodedata", + "unittest", + "urllib", + "uu", + "uuid", + "venv", + "warnings", + "wave", + "weakref", + "webbrowser", + "winreg", + "winsound", + "wsgiref", + "xdrlib", + "xml", + "xmlrpc", + "zipapp", + "zipfile", + "zipimport", + "zlib", + "zoneinfo", +} diff --git a/WebCrawler/venv/Lib/site-packages/isort/utils.py b/WebCrawler/venv/Lib/site-packages/isort/utils.py new file mode 100644 index 0000000..63b5199 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/isort/utils.py @@ -0,0 +1,16 @@ +import os +import sys + + +def exists_case_sensitive(path: str) -> bool: + """Returns if the given path exists and also matches the case on Windows. + + When finding files that can be imported, it is important for the cases to match because while + file os.path.exists("module.py") and os.path.exists("MODULE.py") both return True on Windows, + Python can only import using the case of the real file. + """ + result = os.path.exists(path) + if (sys.platform.startswith("win") or sys.platform == "darwin") and result: # pragma: no cover + directory, basename = os.path.split(path) + result = basename in os.listdir(directory) + return result diff --git a/WebCrawler/venv/Lib/site-packages/isort/wrap.py b/WebCrawler/venv/Lib/site-packages/isort/wrap.py new file mode 100644 index 0000000..11542fa --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/isort/wrap.py @@ -0,0 +1,131 @@ +import copy +import re +from typing import List, Optional, Sequence + +from .settings import DEFAULT_CONFIG, Config +from .wrap_modes import WrapModes as Modes +from .wrap_modes import formatter_from_string + + +def import_statement( + import_start: str, + from_imports: List[str], + comments: Sequence[str] = (), + line_separator: str = "\n", + config: Config = DEFAULT_CONFIG, + multi_line_output: Optional[Modes] = None, +) -> str: + """Returns a multi-line wrapped form of the provided from import statement.""" + formatter = formatter_from_string((multi_line_output or config.multi_line_output).name) + dynamic_indent = " " * (len(import_start) + 1) + indent = config.indent + line_length = config.wrap_length or config.line_length + statement = formatter( + statement=import_start, + imports=copy.copy(from_imports), + white_space=dynamic_indent, + indent=indent, + line_length=line_length, + comments=comments, + line_separator=line_separator, + comment_prefix=config.comment_prefix, + include_trailing_comma=config.include_trailing_comma, + remove_comments=config.ignore_comments, + ) + if config.balanced_wrapping: + lines = statement.split(line_separator) + line_count = len(lines) + if len(lines) > 1: + minimum_length = min(len(line) for line in lines[:-1]) + else: + minimum_length = 0 + new_import_statement = statement + while len(lines[-1]) < minimum_length and len(lines) == line_count and line_length > 10: + statement = new_import_statement + line_length -= 1 + new_import_statement = formatter( + statement=import_start, + imports=copy.copy(from_imports), + white_space=dynamic_indent, + indent=indent, + line_length=line_length, + comments=comments, + line_separator=line_separator, + comment_prefix=config.comment_prefix, + include_trailing_comma=config.include_trailing_comma, + remove_comments=config.ignore_comments, + ) + lines = new_import_statement.split(line_separator) + if statement.count(line_separator) == 0: + return _wrap_line(statement, line_separator, config) + return statement + + +def line(content: str, line_separator: str, config: Config = DEFAULT_CONFIG) -> str: + """Returns a line wrapped to the specified line-length, if possible.""" + wrap_mode = config.multi_line_output + if len(content) > config.line_length and wrap_mode != Modes.NOQA: # type: ignore + line_without_comment = content + comment = None + if "#" in content: + line_without_comment, comment = content.split("#", 1) + for splitter in ("import ", ".", "as "): + exp = r"\b" + re.escape(splitter) + r"\b" + if re.search(exp, line_without_comment) and not line_without_comment.strip().startswith( + splitter + ): + line_parts = re.split(exp, line_without_comment) + if comment and not (config.use_parentheses and "noqa" in comment): + _comma_maybe = ( + "," if (config.include_trailing_comma and config.use_parentheses) else "" + ) + line_parts[ + -1 + ] = f"{line_parts[-1].strip()}{_comma_maybe}{config.comment_prefix}{comment}" + next_line = [] + while (len(content) + 2) > ( + config.wrap_length or config.line_length + ) and line_parts: + next_line.append(line_parts.pop()) + content = splitter.join(line_parts) + if not content: + content = next_line.pop() + + cont_line = _wrap_line( + config.indent + splitter.join(next_line).lstrip(), line_separator, config + ) + if config.use_parentheses: + if splitter == "as ": + output = f"{content}{splitter}{cont_line.lstrip()}" + else: + _comma = "," if config.include_trailing_comma and not comment else "" + if wrap_mode in ( + Modes.VERTICAL_HANGING_INDENT, # type: ignore + Modes.VERTICAL_GRID_GROUPED, # type: ignore + ): + _separator = line_separator + else: + _separator = "" + _comment = "" + if comment and "noqa" in comment: + _comment = f"{config.comment_prefix}{comment}" + cont_line = cont_line.rstrip() + _comma = "," if config.include_trailing_comma else "" + output = ( + f"{content}{splitter}({_comment}" + f"{line_separator}{cont_line}{_comma}{_separator})" + ) + lines = output.split(line_separator) + if config.comment_prefix in lines[-1] and lines[-1].endswith(")"): + content, comment = lines[-1].split(config.comment_prefix, 1) + lines[-1] = content + ")" + config.comment_prefix + comment[:-1] + return line_separator.join(lines) + return f"{content}{splitter}\\{line_separator}{cont_line}" + elif len(content) > config.line_length and wrap_mode == Modes.NOQA: # type: ignore + if "# NOQA" not in content: + return f"{content}{config.comment_prefix} NOQA" + + return content + + +_wrap_line = line diff --git a/WebCrawler/venv/Lib/site-packages/isort/wrap_modes.py b/WebCrawler/venv/Lib/site-packages/isort/wrap_modes.py new file mode 100644 index 0000000..02b7930 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/isort/wrap_modes.py @@ -0,0 +1,325 @@ +"""Defines all wrap modes that can be used when outputting formatted imports""" +import enum +from inspect import signature +from typing import Any, Callable, Dict, List + +import isort.comments + +_wrap_modes: Dict[str, Callable[[Any], str]] = {} + + +def from_string(value: str) -> "WrapModes": + return getattr(WrapModes, str(value), None) or WrapModes(int(value)) + + +def formatter_from_string(name: str): + return _wrap_modes.get(name.upper(), grid) + + +def _wrap_mode_interface( + statement: str, + imports: List[str], + white_space: str, + indent: str, + line_length: int, + comments: List[str], + line_separator: str, + comment_prefix: str, + include_trailing_comma: bool, + remove_comments: bool, +) -> str: + """Defines the common interface used by all wrap mode functions""" + return "" + + +def _wrap_mode(function): + """Registers an individual wrap mode. Function name and order are significant and used for + creating enum. + """ + _wrap_modes[function.__name__.upper()] = function + function.__signature__ = signature(_wrap_mode_interface) + function.__annotations__ = _wrap_mode_interface.__annotations__ + return function + + +@_wrap_mode +def grid(**interface): + if not interface["imports"]: + return "" + + interface["statement"] += "(" + interface["imports"].pop(0) + while interface["imports"]: + next_import = interface["imports"].pop(0) + next_statement = isort.comments.add_to_line( + interface["comments"], + interface["statement"] + ", " + next_import, + removed=interface["remove_comments"], + comment_prefix=interface["comment_prefix"], + ) + if ( + len(next_statement.split(interface["line_separator"])[-1]) + 1 + > interface["line_length"] + ): + lines = [f"{interface['white_space']}{next_import.split(' ')[0]}"] + for part in next_import.split(" ")[1:]: + new_line = f"{lines[-1]} {part}" + if len(new_line) + 1 > interface["line_length"]: + lines.append(f"{interface['white_space']}{part}") + else: + lines[-1] = new_line + next_import = interface["line_separator"].join(lines) + interface["statement"] = ( + isort.comments.add_to_line( + interface["comments"], + f"{interface['statement']},", + removed=interface["remove_comments"], + comment_prefix=interface["comment_prefix"], + ) + + f"{interface['line_separator']}{next_import}" + ) + interface["comments"] = [] + else: + interface["statement"] += ", " + next_import + return interface["statement"] + ("," if interface["include_trailing_comma"] else "") + ")" + + +@_wrap_mode +def vertical(**interface): + if not interface["imports"]: + return "" + + first_import = ( + isort.comments.add_to_line( + interface["comments"], + interface["imports"].pop(0) + ",", + removed=interface["remove_comments"], + comment_prefix=interface["comment_prefix"], + ) + + interface["line_separator"] + + interface["white_space"] + ) + + _imports = ("," + interface["line_separator"] + interface["white_space"]).join( + interface["imports"] + ) + _comma_maybe = "," if interface["include_trailing_comma"] else "" + return f"{interface['statement']}({first_import}{_imports}{_comma_maybe})" + + +def _hanging_indent_common(use_parentheses=False, **interface): + if not interface["imports"]: + return "" + line_length_limit = interface["line_length"] - (1 if use_parentheses else 3) + + def end_line(line): + if use_parentheses: + return line + if not line.endswith(" "): + line += " " + return line + "\\" + + if use_parentheses: + interface["statement"] += "(" + next_import = interface["imports"].pop(0) + next_statement = interface["statement"] + next_import + # Check for first import + if len(next_statement) > line_length_limit: + next_statement = ( + isort.comments.add_to_line( + interface["comments"], + end_line(interface["statement"]), + removed=interface["remove_comments"], + comment_prefix=interface["comment_prefix"], + ) + + f"{interface['line_separator']}{interface['indent']}{next_import}" + ) + interface["comments"] = [] + interface["statement"] = next_statement + while interface["imports"]: + next_import = interface["imports"].pop(0) + next_statement = isort.comments.add_to_line( + interface["comments"], + interface["statement"] + ", " + next_import, + removed=interface["remove_comments"], + comment_prefix=interface["comment_prefix"], + ) + current_line = next_statement.split(interface["line_separator"])[-1] + if len(current_line) > line_length_limit: + next_statement = ( + isort.comments.add_to_line( + interface["comments"], + end_line(interface["statement"] + ","), + removed=interface["remove_comments"], + comment_prefix=interface["comment_prefix"], + ) + + f"{interface['line_separator']}{interface['indent']}{next_import}" + ) + interface["comments"] = [] + interface["statement"] = next_statement + _comma_maybe = "," if interface["include_trailing_comma"] else "" + _close_parentheses_maybe = ")" if use_parentheses else "" + return interface["statement"] + _comma_maybe + _close_parentheses_maybe + + +@_wrap_mode +def hanging_indent(**interface): + return _hanging_indent_common(use_parentheses=False, **interface) + + +@_wrap_mode +def vertical_hanging_indent(**interface): + _line_with_comments = isort.comments.add_to_line( + interface["comments"], + "", + removed=interface["remove_comments"], + comment_prefix=interface["comment_prefix"], + ) + _imports = ("," + interface["line_separator"] + interface["indent"]).join(interface["imports"]) + _comma_maybe = "," if interface["include_trailing_comma"] else "" + return ( + f"{interface['statement']}({_line_with_comments}{interface['line_separator']}" + f"{interface['indent']}{_imports}{_comma_maybe}{interface['line_separator']})" + ) + + +def _vertical_grid_common(need_trailing_char: bool, **interface): + if not interface["imports"]: + return "" + + interface["statement"] += ( + isort.comments.add_to_line( + interface["comments"], + "(", + removed=interface["remove_comments"], + comment_prefix=interface["comment_prefix"], + ) + + interface["line_separator"] + + interface["indent"] + + interface["imports"].pop(0) + ) + while interface["imports"]: + next_import = interface["imports"].pop(0) + next_statement = f"{interface['statement']}, {next_import}" + current_line_length = len(next_statement.split(interface["line_separator"])[-1]) + if interface["imports"] or need_trailing_char: + # If we have more interface["imports"] we need to account for a comma after this import + # We might also need to account for a closing ) we're going to add. + current_line_length += 1 + if current_line_length > interface["line_length"]: + next_statement = ( + f"{interface['statement']},{interface['line_separator']}" + f"{interface['indent']}{next_import}" + ) + interface["statement"] = next_statement + if interface["include_trailing_comma"]: + interface["statement"] += "," + return interface["statement"] + + +@_wrap_mode +def vertical_grid(**interface) -> str: + return _vertical_grid_common(need_trailing_char=True, **interface) + ")" + + +@_wrap_mode +def vertical_grid_grouped(**interface): + return ( + _vertical_grid_common(need_trailing_char=True, **interface) + + interface["line_separator"] + + ")" + ) + + +@_wrap_mode +def vertical_grid_grouped_no_comma(**interface): + return ( + _vertical_grid_common(need_trailing_char=False, **interface) + + interface["line_separator"] + + ")" + ) + + +@_wrap_mode +def noqa(**interface): + _imports = ", ".join(interface["imports"]) + retval = f"{interface['statement']}{_imports}" + comment_str = " ".join(interface["comments"]) + if interface["comments"]: + if ( + len(retval) + len(interface["comment_prefix"]) + 1 + len(comment_str) + <= interface["line_length"] + ): + return f"{retval}{interface['comment_prefix']} {comment_str}" + elif "NOQA" in interface["comments"]: + return f"{retval}{interface['comment_prefix']} {comment_str}" + else: + return f"{retval}{interface['comment_prefix']} NOQA {comment_str}" + else: + if len(retval) <= interface["line_length"]: + return retval + else: + return f"{retval}{interface['comment_prefix']} NOQA" + + +@_wrap_mode +def vertical_hanging_indent_bracket(**interface): + if not interface["imports"]: + return "" + statement = vertical_hanging_indent(**interface) + return f'{statement[:-1]}{interface["indent"]})' + + +@_wrap_mode +def vertical_prefix_from_module_import(**interface): + if not interface["imports"]: + return "" + + prefix_statement = interface["statement"] + output_statement = prefix_statement + interface["imports"].pop(0) + comments = interface["comments"] + + statement = output_statement + statement_with_comments = "" + for next_import in interface["imports"]: + statement = statement + ", " + next_import + statement_with_comments = isort.comments.add_to_line( + comments, + statement, + removed=interface["remove_comments"], + comment_prefix=interface["comment_prefix"], + ) + if ( + len(statement_with_comments.split(interface["line_separator"])[-1]) + 1 + > interface["line_length"] + ): + statement = ( + isort.comments.add_to_line( + comments, + output_statement, + removed=interface["remove_comments"], + comment_prefix=interface["comment_prefix"], + ) + + f"{interface['line_separator']}{prefix_statement}{next_import}" + ) + comments = [] + output_statement = statement + + if comments and statement_with_comments: + output_statement = statement_with_comments + return output_statement + + +@_wrap_mode +def hanging_indent_with_parentheses(**interface): + return _hanging_indent_common(use_parentheses=True, **interface) + + +@_wrap_mode +def backslash_grid(**interface): + interface["indent"] = interface["white_space"][:-1] + return _hanging_indent_common(use_parentheses=False, **interface) + + +WrapModes = enum.Enum( # type: ignore + "WrapModes", {wrap_mode: index for index, wrap_mode in enumerate(_wrap_modes.keys())} +) diff --git a/WebCrawler/venv/Lib/site-packages/itemadapter-0.2.0.dist-info/INSTALLER b/WebCrawler/venv/Lib/site-packages/itemadapter-0.2.0.dist-info/INSTALLER new file mode 100644 index 0000000..a1b589e --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/itemadapter-0.2.0.dist-info/INSTALLER @@ -0,0 +1 @@ +pip diff --git a/WebCrawler/venv/Lib/site-packages/itemadapter-0.2.0.dist-info/LICENSE b/WebCrawler/venv/Lib/site-packages/itemadapter-0.2.0.dist-info/LICENSE new file mode 100644 index 0000000..ac83e7d --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/itemadapter-0.2.0.dist-info/LICENSE @@ -0,0 +1,26 @@ +Copyright 2020 Eugenio Lacuesta + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, +this list of conditions and the following disclaimer in the documentation and/or +other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its contributors +may be used to endorse or promote products derived from this software without +specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/WebCrawler/venv/Lib/site-packages/itemadapter-0.2.0.dist-info/METADATA b/WebCrawler/venv/Lib/site-packages/itemadapter-0.2.0.dist-info/METADATA new file mode 100644 index 0000000..d844e14 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/itemadapter-0.2.0.dist-info/METADATA @@ -0,0 +1,424 @@ +Metadata-Version: 2.1 +Name: itemadapter +Version: 0.2.0 +Summary: Common interface for data container classes +Home-page: https://github.com/scrapy/itemadapter +Author: Eugenio Lacuesta +Author-email: eugenio.lacuesta@gmail.com +License: BSD +Platform: UNKNOWN +Classifier: Development Status :: 3 - Alpha +Classifier: License :: OSI Approved :: BSD License +Classifier: Programming Language :: Python +Classifier: Programming Language :: Python :: 3.6 +Classifier: Programming Language :: Python :: 3.7 +Classifier: Programming Language :: Python :: 3.8 +Classifier: Framework :: Scrapy +Classifier: Intended Audience :: Developers +Classifier: Topic :: Internet :: WWW/HTTP +Classifier: Topic :: Software Development :: Libraries :: Application Frameworks +Classifier: Topic :: Software Development :: Libraries :: Python Modules +Requires-Python: >=3.6 +Description-Content-Type: text/markdown + +# itemadapter +[![version](https://img.shields.io/pypi/v/itemadapter.svg)](https://pypi.python.org/pypi/itemadapter) +[![pyversions](https://img.shields.io/pypi/pyversions/itemadapter.svg)](https://pypi.python.org/pypi/itemadapter) +[![actions](https://github.com/scrapy/itemadapter/workflows/Build/badge.svg)](https://github.com/scrapy/itemadapter/actions) +[![codecov](https://codecov.io/gh/scrapy/itemadapter/branch/master/graph/badge.svg)](https://codecov.io/gh/scrapy/itemadapter) + + +The `ItemAdapter` class is a wrapper for data container objects, providing a +common interface to handle objects of different types in an uniform manner, +regardless of their underlying implementation. + +Currently supported types are: + +* [`dict`](https://docs.python.org/3/library/stdtypes.html#dict) +* [`scrapy.item.Item`](https://docs.scrapy.org/en/latest/topics/items.html#scrapy.item.Item) +* [`dataclass`](https://docs.python.org/3/library/dataclasses.html)-based classes +* [`attrs`](https://www.attrs.org)-based classes + + +## Requirements + +* Python 3.6+ +* [`scrapy`](https://scrapy.org/): optional, needed to interact with `scrapy` items +* `dataclasses` ([stdlib](https://docs.python.org/3/library/dataclasses.html) in Python 3.7+, + or its [backport](https://pypi.org/project/dataclasses/) in Python 3.6): optional, needed + to interact with `dataclass`-based items +* [`attrs`](https://pypi.org/project/attrs/): optional, needed to interact with `attrs`-based items + + +## Installation + +`itemadapter` is available on [`PyPI`](https://pypi.python.org/pypi/itemadapter), it can be installed with `pip`: + +``` +pip install itemadapter +``` + + +## License + +`itemadapter` is distributed under a [BSD-3](https://opensource.org/licenses/BSD-3-Clause) license. + + +## Basic usage + +The following is a simple example using a `dataclass` object. +Consider the following type definition: + +```python +>>> from dataclasses import dataclass +>>> from itemadapter import ItemAdapter, is_item +>>> @dataclass +... class InventoryItem: +... name: str +... price: float +... stock: int +>>> +``` + +The `ItemAdapter` object can be treated much like a dictionary: + +```python +>>> obj = InventoryItem(name='foo', price=20.5, stock=10) +>>> is_item(obj) +True +>>> adapter = ItemAdapter(obj) +>>> len(adapter) +3 +>>> adapter["name"] +'foo' +>>> adapter.get("price") +20.5 +>>> +``` + +The wrapped object is modified in-place: +```python +>>> adapter["name"] = "bar" +>>> adapter.update({"price": 12.7, "stock": 9}) +>>> adapter.item +InventoryItem(name='bar', price=12.7, stock=9) +>>> adapter.item is obj +True +>>> +``` + +### Converting to dict + +The `ItemAdapter` class provides the `asdict` method, which converts +nested items recursively. Consider the following example: + +```python +>>> from dataclasses import dataclass +>>> from itemadapter import ItemAdapter +>>> @dataclass +... class Price: +... value: int +... currency: str +>>> @dataclass +... class Product: +... name: str +... price: Price +>>> +``` + +```python +>>> item = Product("Stuff", Price(42, "UYU")) +>>> adapter = ItemAdapter(item) +>>> adapter.asdict() +{'name': 'Stuff', 'price': {'value': 42, 'currency': 'UYU'}} +>>> +``` + +Note that just passing an adapter object to the `dict` built-in also works, +but it doesn't traverse the object recursively converting nested items: + +```python +>>> dict(adapter) +{'name': 'Stuff', 'price': Price(value=42, currency='UYU')} +>>> +``` + + +## API + +### Built-in adapters + +The following adapters are included by default: + +* `itemadapter.adapter.ScrapyItemAdapter`: handles `Scrapy` items +* `itemadapter.adapter.DictAdapter`: handles `Python` dictionaries +* `itemadapter.adapter.DataclassAdapter`: handles `dataclass` objects +* `itemadapter.adapter.AttrsAdapter`: handles `attrs` objects + +### `ItemAdapter` class + +_class `itemadapter.adapter.ItemAdapter(item: Any)`_ + +This is the main entrypoint for the package. Tipically, user code +wraps an item using this class, and proceeds to handle it with the provided interface. +`ItemAdapter` implements the +[`MutableMapping`](https://docs.python.org/3/library/collections.abc.html#collections.abc.MutableMapping) +interface, providing a `dict`-like API to manipulate data for the object it wraps +(which is modified in-place). + +Some additional methods are available: + +`get_field_meta(field_name: str) -> MappingProxyType` + +Return a [`types.MappingProxyType`](https://docs.python.org/3/library/types.html#types.MappingProxyType) +object, which is a read-only mapping with metadata about the given field. If the item class does not +support field metadata, or there is no metadata for the given field, an empty object is returned. + +The returned value is taken from the following sources, depending on the item type: + +* [`scrapy.item.Field`](https://docs.scrapy.org/en/latest/topics/items.html#item-fields) +for `scrapy.item.Item`s +* [`dataclasses.field.metadata`](https://docs.python.org/3/library/dataclasses.html#dataclasses.field) + for `dataclass`-based items +* [`attr.Attribute.metadata`](https://www.attrs.org/en/stable/examples.html#metadata) + for `attrs`-based items + +`field_names() -> collections.abc.KeysView` + +Return a [keys view](https://docs.python.org/3/library/collections.abc.html#collections.abc.KeysView) +with the names of all the defined fields for the item. + +`asdict() -> dict` + +Return a `dict` object with the contents of the adapter. This works slightly different than +calling `dict(adapter)`, because it's applied recursively to nested items (if there are any). + +### `is_item` function + +_`itemadapter.utils.is_item(obj: Any) -> bool`_ + +Return `True` if the given object belongs to (at least) one of the supported types, +`False` otherwise. + +### `get_field_meta_from_class` function + +_`itemadapter.utils.get_field_meta_from_class(item_class: type, field_name: str) -> types.MappingProxyType`_ + +Given an item class and a field name, return a +[`MappingProxyType`](https://docs.python.org/3/library/types.html#types.MappingProxyType) +object, which is a read-only mapping with metadata about the given field. If the item class does not +support field metadata, or there is no metadata for the given field, an empty object is returned. + + +## Metadata support + +`scrapy.item.Item`, `dataclass` and `attrs` objects allow the definition of +arbitrary field metadata. This can be accessed through a +[`MappingProxyType`](https://docs.python.org/3/library/types.html#types.MappingProxyType) +object, which can be retrieved from an item instance with the +`itemadapter.adapter.ItemAdapter.get_field_meta` method, or from an item class +with the `itemadapter.utils.get_field_meta_from_class` function. +The definition procedure depends on the underlying type. + +#### `scrapy.item.Item` objects + +```python +>>> from scrapy.item import Item, Field +>>> from itemadapter import ItemAdapter +>>> class InventoryItem(Item): +... name = Field(serializer=str) +... value = Field(serializer=int, limit=100) +... +>>> adapter = ItemAdapter(InventoryItem(name="foo", value=10)) +>>> adapter.get_field_meta("name") +mappingproxy({'serializer': }) +>>> adapter.get_field_meta("value") +mappingproxy({'serializer': , 'limit': 100}) +>>> +``` + +#### `dataclass` objects + +```python +>>> from dataclasses import dataclass, field +>>> @dataclass +... class InventoryItem: +... name: str = field(metadata={"serializer": str}) +... value: int = field(metadata={"serializer": int, "limit": 100}) +... +>>> adapter = ItemAdapter(InventoryItem(name="foo", value=10)) +>>> adapter.get_field_meta("name") +mappingproxy({'serializer': }) +>>> adapter.get_field_meta("value") +mappingproxy({'serializer': , 'limit': 100}) +>>> +``` + +#### `attrs` objects + +```python +>>> import attr +>>> @attr.s +... class InventoryItem: +... name = attr.ib(metadata={"serializer": str}) +... value = attr.ib(metadata={"serializer": int, "limit": 100}) +... +>>> adapter = ItemAdapter(InventoryItem(name="foo", value=10)) +>>> adapter.get_field_meta("name") +mappingproxy({'serializer': }) +>>> adapter.get_field_meta("value") +mappingproxy({'serializer': , 'limit': 100}) +>>> +``` + + +## Extending `itemadapter` + +This package allows to handle arbitrary item classes, by implementing an adapter interface: + +_class `itemadapter.adapter.AdapterInterface(item: Any)`_ + +Abstract Base Class for adapters. An adapter that handles a specific type of item must +inherit from this class and implement the abstract methods defined on it. `AdapterInterface` +inherits from [`collections.abc.MutableMapping`](https://docs.python.org/3/library/collections.abc.html#collections.abc.MutableMapping), +so all methods from the `MutableMapping` class must be implemented as well. + +* _class method `is_item(cls, item: Any) -> bool`_ + + Return `True` if the adapter can handle the given item, `False` otherwise. Abstract (mandatory). + +* _method `get_field_meta(self, field_name: str) -> types.MappingProxyType`_ + + Return metadata for the given field name, if available. + By default, this method returns an empty `MappingProxyType` object. Please supply your + own method definition if you want to handle field metadata based on custom logic. + See the [section on metadata support](#metadata-support) for additional information. + +* _method `field_names(self) -> collections.abc.KeysView`_: + + Return a [dynamic view](https://docs.python.org/3/library/collections.abc.html#collections.abc.KeysView) + of the item's field names. By default, this method returns the result of calling `keys()` on + the current adapter, i.e., its return value depends on the implementation of the methods from the + `MutableMapping` interface (more specifically, it depends on the return value of `__iter__`). + + You might want to override this method if you want a way to get all fields for an item, whether or not + they are populated. For instance, Scrapy uses this method to define column names when exporting items to CSV. + +### Registering an adapter + +The `itemadapter.adapter.ItemAdapter` class keeps the registered adapters in its `ADAPTER_CLASSES` +class attribute. This is a +[`collections.deque`](https://docs.python.org/3/library/collections.html#collections.deque) +object, allowing to efficiently add new adapters elements to both ends. + +The order in which the adapters are registered is important. When an `ItemAdapter` object is +created for a specific item, the registered adapters are traversed in order and the first class +to return `True` for the `is_item` class method is used for all subsequent operations. + +**Example** +```python +>>> from itemadapter.adapter import AdapterInterface, ItemAdapter +>>> from tests.test_interface import BaseFakeItemAdapter, FakeItemClass +>>> +>>> ItemAdapter.ADAPTER_CLASSES.appendleft(BaseFakeItemAdapter) +>>> item = FakeItemClass() +>>> adapter = ItemAdapter(item) +>>> adapter + +>>> +``` + + +## More examples + +### `scrapy.item.Item` objects + +```python +>>> from scrapy.item import Item, Field +>>> from itemadapter import ItemAdapter +>>> class InventoryItem(Item): +... name = Field() +... price = Field() +... +>>> item = InventoryItem(name="foo", price=10) +>>> adapter = ItemAdapter(item) +>>> adapter.item is item +True +>>> adapter["name"] +'foo' +>>> adapter["name"] = "bar" +>>> adapter["price"] = 5 +>>> item +{'name': 'bar', 'price': 5} +>>> +``` + +### `dict` + +```python +>>> from itemadapter import ItemAdapter +>>> item = dict(name="foo", price=10) +>>> adapter = ItemAdapter(item) +>>> adapter.item is item +True +>>> adapter["name"] +'foo' +>>> adapter["name"] = "bar" +>>> adapter["price"] = 5 +>>> item +{'name': 'bar', 'price': 5} +>>> +``` + +### `dataclass` objects + +```python +>>> from dataclasses import dataclass +>>> from itemadapter import ItemAdapter +>>> @dataclass +... class InventoryItem: +... name: str +... price: int +... +>>> item = InventoryItem(name="foo", price=10) +>>> adapter = ItemAdapter(item) +>>> adapter.item is item +True +>>> adapter["name"] +'foo' +>>> adapter["name"] = "bar" +>>> adapter["price"] = 5 +>>> item +InventoryItem(name='bar', price=5) +>>> +``` + +### `attrs` objects + +```python +>>> import attr +>>> from itemadapter import ItemAdapter +>>> @attr.s +... class InventoryItem: +... name = attr.ib() +... price = attr.ib() +... +>>> item = InventoryItem(name="foo", price=10) +>>> adapter = ItemAdapter(item) +>>> adapter.item is item +True +>>> adapter["name"] +'foo' +>>> adapter["name"] = "bar" +>>> adapter["price"] = 5 +>>> item +InventoryItem(name='bar', price=5) +>>> +``` + + +## Changelog + +See the [full changelog](Changelog.md) + + diff --git a/WebCrawler/venv/Lib/site-packages/itemadapter-0.2.0.dist-info/RECORD b/WebCrawler/venv/Lib/site-packages/itemadapter-0.2.0.dist-info/RECORD new file mode 100644 index 0000000..f92f99c --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/itemadapter-0.2.0.dist-info/RECORD @@ -0,0 +1,12 @@ +itemadapter-0.2.0.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4 +itemadapter-0.2.0.dist-info/LICENSE,sha256=irgaybyKbgi_FWAP_B5YejcPBH0Amj-mtT0O6bmr0A0,1461 +itemadapter-0.2.0.dist-info/METADATA,sha256=tJMIuBXHVvit5MYIgzEpp1XBbFP3LityQ4NCKtm4ylU,13368 +itemadapter-0.2.0.dist-info/RECORD,, +itemadapter-0.2.0.dist-info/WHEEL,sha256=EVRjI69F5qVjm_YgqcTXPnTAv3BfSUr0WVAHuSP3Xoo,92 +itemadapter-0.2.0.dist-info/top_level.txt,sha256=kI6F__diTnM_i5NLkCir6a1W1a1SgWdZLPsj2_2qoYU,12 +itemadapter/__init__.py,sha256=5h_7vLeFSOj7ZfyaLBdUV3-1BIwN7zNQkCrDbSXKFEc,139 +itemadapter/__pycache__/__init__.cpython-39.pyc,, +itemadapter/__pycache__/adapter.cpython-39.pyc,, +itemadapter/__pycache__/utils.cpython-39.pyc,, +itemadapter/adapter.py,sha256=G43qeMN1ENxzEmobgV8_DXf0gxzeYCpYgqSuY6pWpwc,8271 +itemadapter/utils.py,sha256=H8yQZcwsTtRp0YGgZerStxEA751PPM3YyBQbuPsz2TA,3733 diff --git a/WebCrawler/venv/Lib/site-packages/itemadapter-0.2.0.dist-info/WHEEL b/WebCrawler/venv/Lib/site-packages/itemadapter-0.2.0.dist-info/WHEEL new file mode 100644 index 0000000..83ff02e --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/itemadapter-0.2.0.dist-info/WHEEL @@ -0,0 +1,5 @@ +Wheel-Version: 1.0 +Generator: bdist_wheel (0.35.1) +Root-Is-Purelib: true +Tag: py3-none-any + diff --git a/WebCrawler/venv/Lib/site-packages/itemadapter-0.2.0.dist-info/top_level.txt b/WebCrawler/venv/Lib/site-packages/itemadapter-0.2.0.dist-info/top_level.txt new file mode 100644 index 0000000..46fdc91 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/itemadapter-0.2.0.dist-info/top_level.txt @@ -0,0 +1 @@ +itemadapter diff --git a/WebCrawler/venv/Lib/site-packages/itemadapter/__init__.py b/WebCrawler/venv/Lib/site-packages/itemadapter/__init__.py new file mode 100644 index 0000000..177072a --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/itemadapter/__init__.py @@ -0,0 +1,5 @@ +from .adapter import ItemAdapter # noqa: F401 +from .utils import get_field_meta_from_class, is_item # noqa: F401 + + +__version__ = "0.2.0" diff --git a/WebCrawler/venv/Lib/site-packages/itemadapter/__pycache__/__init__.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/itemadapter/__pycache__/__init__.cpython-39.pyc new file mode 100644 index 0000000..47407ad Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/itemadapter/__pycache__/__init__.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/itemadapter/__pycache__/adapter.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/itemadapter/__pycache__/adapter.cpython-39.pyc new file mode 100644 index 0000000..8101c46 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/itemadapter/__pycache__/adapter.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/itemadapter/__pycache__/utils.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/itemadapter/__pycache__/utils.cpython-39.pyc new file mode 100644 index 0000000..4a04cd2 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/itemadapter/__pycache__/utils.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/itemadapter/adapter.py b/WebCrawler/venv/Lib/site-packages/itemadapter/adapter.py new file mode 100644 index 0000000..ac2f9ee --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/itemadapter/adapter.py @@ -0,0 +1,264 @@ +from abc import abstractmethod, ABCMeta +from collections import deque +from collections.abc import KeysView, MutableMapping +from types import MappingProxyType +from typing import Any, Iterator + +from itemadapter.utils import ( + is_attrs_instance, + is_dataclass_instance, + is_item, + is_scrapy_item, +) + + +__all__ = [ + "AdapterInterface", + "AttrsAdapter", + "DataclassAdapter", + "DictAdapter", + "ItemAdapter", + "ScrapyItemAdapter", +] + + +class AdapterInterface(MutableMapping, metaclass=ABCMeta): + """ + Abstract Base Class for adapters. + + An adapter that handles a specific type of item should inherit from this + class and implement the abstract methods defined here, plus the + abtract methods inherited from the MutableMapping base class. + """ + + def __init__(self, item: Any) -> None: + self.item = item + + @classmethod + @abstractmethod + def is_item(cls, item: Any) -> bool: + """ + Return True if the adapter can handle the given item, False otherwise + """ + raise NotImplementedError() + + def get_field_meta(self, field_name: str) -> MappingProxyType: + """ + Return metadata for the given field name, if available + """ + return MappingProxyType({}) + + def field_names(self) -> KeysView: + """ + Return a dynamic view of the item's field names + """ + return self.keys() # type: ignore + + +class _MixinAttrsDataclassAdapter: + + _fields_dict: dict + item: Any + + def get_field_meta(self, field_name: str) -> MappingProxyType: + return self._fields_dict[field_name].metadata # type: ignore + + def field_names(self) -> KeysView: + return KeysView(self._fields_dict) + + def __getitem__(self, field_name: str) -> Any: + if field_name in self._fields_dict: + return getattr(self.item, field_name) + raise KeyError(field_name) + + def __setitem__(self, field_name: str, value: Any) -> None: + if field_name in self._fields_dict: + setattr(self.item, field_name, value) + else: + raise KeyError(f"{self.item.__class__.__name__} does not support field: {field_name}") + + def __delitem__(self, field_name: str) -> None: + if field_name in self._fields_dict: + try: + delattr(self.item, field_name) + except AttributeError: + raise KeyError(field_name) + else: + raise KeyError(f"{self.item.__class__.__name__} does not support field: {field_name}") + + def __iter__(self) -> Iterator: + return iter(attr for attr in self._fields_dict if hasattr(self.item, attr)) + + def __len__(self) -> int: + return len(list(iter(self))) + + +class AttrsAdapter(_MixinAttrsDataclassAdapter, AdapterInterface): + def __init__(self, item: Any) -> None: + super().__init__(item) + import attr + + # store a reference to the item's fields to avoid O(n) lookups and O(n^2) traversals + self._fields_dict = attr.fields_dict(self.item.__class__) + + @classmethod + def is_item(cls, item: Any) -> bool: + return is_attrs_instance(item) + + +class DataclassAdapter(_MixinAttrsDataclassAdapter, AdapterInterface): + def __init__(self, item: Any) -> None: + super().__init__(item) + import dataclasses + + # store a reference to the item's fields to avoid O(n) lookups and O(n^2) traversals + self._fields_dict = {field.name: field for field in dataclasses.fields(self.item)} + + @classmethod + def is_item(cls, item: Any) -> bool: + return is_dataclass_instance(item) + + +class _MixinDictScrapyItemAdapter: + + _fields_dict: dict + item: Any + + def __getitem__(self, field_name: str) -> Any: + return self.item[field_name] + + def __setitem__(self, field_name: str, value: Any) -> None: + self.item[field_name] = value + + def __delitem__(self, field_name: str) -> None: + del self.item[field_name] + + def __iter__(self) -> Iterator: + return iter(self.item) + + def __len__(self) -> int: + return len(self.item) + + +class DictAdapter(_MixinDictScrapyItemAdapter, AdapterInterface): + @classmethod + def is_item(cls, item: Any) -> bool: + return isinstance(item, dict) + + def get_field_meta(self, field_name: str) -> MappingProxyType: + return MappingProxyType({}) + + def field_names(self) -> KeysView: + return KeysView(self.item) + + +class ScrapyItemAdapter(_MixinDictScrapyItemAdapter, AdapterInterface): + @classmethod + def is_item(cls, item: Any) -> bool: + return is_scrapy_item(item) + + def get_field_meta(self, field_name: str) -> MappingProxyType: + return MappingProxyType(self.item.fields[field_name]) + + def field_names(self) -> KeysView: + return KeysView(self.item.fields) + + +class ItemAdapter(MutableMapping): + """ + Wrapper class to interact with data container objects. It provides a common interface + to extract and set data without having to take the object's type into account. + """ + + ADAPTER_CLASSES = deque( + [ + ScrapyItemAdapter, + DictAdapter, + DataclassAdapter, + AttrsAdapter, + ] + ) + + def __init__(self, item: Any) -> None: + self.adapter_class = None + for cls in self.ADAPTER_CLASSES: + if cls.is_item(item): + self.adapter = cls(item) # type: ignore + break + else: + raise TypeError(f"No adapter found for objects of type: {type(item)} ({item})") + + @classmethod + def is_item(self, item: Any) -> bool: + for cls in self.ADAPTER_CLASSES: + if cls.is_item(item): + return True + return False + + @property + def item(self) -> Any: + return self.adapter.item + + def __repr__(self) -> str: + values = ", ".join(["%s=%r" % (key, value) for key, value in self.items()]) + return f"" + + def __getitem__(self, field_name: str) -> Any: + return self.adapter.__getitem__(field_name) + + def __setitem__(self, field_name: str, value: Any) -> None: + self.adapter.__setitem__(field_name, value) + + def __delitem__(self, field_name: str) -> None: + self.adapter.__delitem__(field_name) + + def __iter__(self) -> Iterator: + return self.adapter.__iter__() + + def __len__(self) -> int: + return self.adapter.__len__() + + def get_field_meta(self, field_name: str) -> MappingProxyType: + """ + Return a read-only mapping with metadata for the given field name. If there is no metadata + for the field, or the wrapped item does not support field metadata, an empty object is + returned. + + Field metadata is taken from different sources, depending on the item type: + * scrapy.item.Item: corresponding scrapy.item.Field object + * dataclass items: "metadata" attribute for the corresponding field + * attrs items: "metadata" attribute for the corresponding field + + The returned value is an instance of types.MappingProxyType, i.e. a dynamic read-only view + of the original mapping, which gets automatically updated if the original mapping changes. + """ + return self.adapter.get_field_meta(field_name) + + def field_names(self) -> KeysView: + """ + Return read-only key view with the names of all the defined fields for the item + """ + return self.adapter.field_names() + + def asdict(self) -> dict: + """ + Return a dict object with the contents of the adapter. This works slightly different than + calling `dict(adapter)`: it's applied recursively to nested items (if there are any). + """ + return {key: _asdict(value) for key, value in self.items()} # type: ignore + + +def _asdict(obj: Any) -> Any: + """ + Helper for ItemAdapter.asdict + """ + if isinstance(obj, dict): + return {key: _asdict(value) for key, value in obj.items()} + elif isinstance(obj, (list, set, tuple)): + return obj.__class__(_asdict(x) for x in obj) + elif isinstance(obj, ItemAdapter): + return obj.asdict() + elif is_item(obj): + return ItemAdapter(obj).asdict() + else: + return obj diff --git a/WebCrawler/venv/Lib/site-packages/itemadapter/utils.py b/WebCrawler/venv/Lib/site-packages/itemadapter/utils.py new file mode 100644 index 0000000..dcc71b3 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/itemadapter/utils.py @@ -0,0 +1,114 @@ +from types import MappingProxyType +from typing import Any + + +def _get_scrapy_item_classes() -> tuple: + try: + import scrapy + except ImportError: + return () + else: + try: + _base_item_cls = getattr(scrapy.item, "_BaseItem", scrapy.item.BaseItem) # deprecated + return (scrapy.item.Item, _base_item_cls) + except AttributeError: + return (scrapy.item.Item,) + + +def _is_dataclass(obj: Any) -> bool: + try: + import dataclasses + except ImportError: + return False + return dataclasses.is_dataclass(obj) + + +def _is_attrs_class(obj: Any) -> bool: + try: + import attr + except ImportError: + return False + return attr.has(obj) + + +def is_dataclass_instance(obj: Any) -> bool: + """ + Return True if the given object is a dataclass object, False otherwise. + + In py36, this function returns False if the "dataclasses" backport is not available. + + Taken from https://docs.python.org/3/library/dataclasses.html#dataclasses.is_dataclass. + """ + return _is_dataclass(obj) and not isinstance(obj, type) + + +def is_attrs_instance(obj: Any) -> bool: + """ + Return True if the given object is a attrs-based object, False otherwise. + """ + return _is_attrs_class(obj) and not isinstance(obj, type) + + +def is_scrapy_item(obj: Any) -> bool: + """ + Return True if the given object is a Scrapy item, False otherwise. + """ + try: + import scrapy + except ImportError: + return False + if isinstance(obj, scrapy.item.Item): + return True + try: + # handle deprecated BaseItem + BaseItem = getattr(scrapy.item, "_BaseItem", scrapy.item.BaseItem) + return isinstance(obj, BaseItem) + except AttributeError: + return False + + +def is_item(obj: Any) -> bool: + """ + Return True if the given object belongs to one of the supported types, False otherwise. + + Alias for ItemAdapter.is_item + """ + from itemadapter.adapter import ItemAdapter + + return ItemAdapter.is_item(obj) + + +def get_field_meta_from_class(item_class: type, field_name: str) -> MappingProxyType: + """ + Return a read-only mapping with metadata for the given field name, within the given item class. + If there is no metadata for the field, or the item class does not support field metadata, + an empty object is returned. + + Field metadata is taken from different sources, depending on the item type: + * scrapy.item.Item: corresponding scrapy.item.Field object + * dataclass items: "metadata" attribute for the corresponding field + * attrs items: "metadata" attribute for the corresponding field + + The returned value is an instance of types.MappingProxyType, i.e. a dynamic read-only view + of the original mapping, which gets automatically updated if the original mapping changes. + """ + if issubclass(item_class, _get_scrapy_item_classes()): + return MappingProxyType(item_class.fields[field_name]) # type: ignore + elif _is_dataclass(item_class): + from dataclasses import fields + + for field in fields(item_class): + if field.name == field_name: + return field.metadata # type: ignore + raise KeyError("%s does not support field: %s" % (item_class.__name__, field_name)) + elif _is_attrs_class(item_class): + from attr import fields_dict + + try: + return fields_dict(item_class)[field_name].metadata # type: ignore + except KeyError: + raise KeyError("%s does not support field: %s" % (item_class.__name__, field_name)) + elif issubclass(item_class, dict): + return MappingProxyType({}) + else: + raise TypeError("%s is not a valid item class" % (item_class,)) diff --git a/WebCrawler/venv/Lib/site-packages/jmespath-0.10.0.dist-info/DESCRIPTION.rst b/WebCrawler/venv/Lib/site-packages/jmespath-0.10.0.dist-info/DESCRIPTION.rst new file mode 100644 index 0000000..c5e916e --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/jmespath-0.10.0.dist-info/DESCRIPTION.rst @@ -0,0 +1,224 @@ +JMESPath +======== + + +.. image:: https://badges.gitter.im/Join Chat.svg + :target: https://gitter.im/jmespath/chat + + +.. image:: https://travis-ci.org/jmespath/jmespath.py.svg?branch=develop + :target: https://travis-ci.org/jmespath/jmespath.py + + +.. image:: https://codecov.io/github/jmespath/jmespath.py/coverage.svg?branch=develop + :target: https://codecov.io/github/jmespath/jmespath.py?branch=develop + + +JMESPath (pronounced "james path") allows you to declaratively specify how to +extract elements from a JSON document. + +For example, given this document:: + + {"foo": {"bar": "baz"}} + +The jmespath expression ``foo.bar`` will return "baz". + +JMESPath also supports: + +Referencing elements in a list. Given the data:: + + {"foo": {"bar": ["one", "two"]}} + +The expression: ``foo.bar[0]`` will return "one". +You can also reference all the items in a list using the ``*`` +syntax:: + + {"foo": {"bar": [{"name": "one"}, {"name": "two"}]}} + +The expression: ``foo.bar[*].name`` will return ["one", "two"]. +Negative indexing is also supported (-1 refers to the last element +in the list). Given the data above, the expression +``foo.bar[-1].name`` will return "two". + +The ``*`` can also be used for hash types:: + + {"foo": {"bar": {"name": "one"}, "baz": {"name": "two"}}} + +The expression: ``foo.*.name`` will return ["one", "two"]. + + +Installation +============ + +You can install JMESPath from pypi with: + +.. code:: bash + + pip install jmespath + + +API +=== + +The ``jmespath.py`` library has two functions +that operate on python data structures. You can use ``search`` +and give it the jmespath expression and the data: + +.. code:: python + + >>> import jmespath + >>> path = jmespath.search('foo.bar', {'foo': {'bar': 'baz'}}) + 'baz' + +Similar to the ``re`` module, you can use the ``compile`` function +to compile the JMESPath expression and use this parsed expression +to perform repeated searches: + +.. code:: python + + >>> import jmespath + >>> expression = jmespath.compile('foo.bar') + >>> expression.search({'foo': {'bar': 'baz'}}) + 'baz' + >>> expression.search({'foo': {'bar': 'other'}}) + 'other' + +This is useful if you're going to use the same jmespath expression to +search multiple documents. This avoids having to reparse the +JMESPath expression each time you search a new document. + +Options +------- + +You can provide an instance of ``jmespath.Options`` to control how +a JMESPath expression is evaluated. The most common scenario for +using an ``Options`` instance is if you want to have ordered output +of your dict keys. To do this you can use either of these options: + +.. code:: python + + >>> import jmespath + >>> jmespath.search('{a: a, b: b}', + ... mydata, + ... jmespath.Options(dict_cls=collections.OrderedDict)) + + + >>> import jmespath + >>> parsed = jmespath.compile('{a: a, b: b}') + >>> parsed.search(mydata, + ... jmespath.Options(dict_cls=collections.OrderedDict)) + + +Custom Functions +~~~~~~~~~~~~~~~~ + +The JMESPath language has numerous +`built-in functions +`__, but it is +also possible to add your own custom functions. Keep in mind that +custom function support in jmespath.py is experimental and the API may +change based on feedback. + +**If you have a custom function that you've found useful, consider submitting +it to jmespath.site and propose that it be added to the JMESPath language.** +You can submit proposals +`here `__. + +To create custom functions: + +* Create a subclass of ``jmespath.functions.Functions``. +* Create a method with the name ``_func_``. +* Apply the ``jmespath.functions.signature`` decorator that indicates + the expected types of the function arguments. +* Provide an instance of your subclass in a ``jmespath.Options`` object. + +Below are a few examples: + +.. code:: python + + import jmespath + from jmespath import functions + + # 1. Create a subclass of functions.Functions. + # The function.Functions base class has logic + # that introspects all of its methods and automatically + # registers your custom functions in its function table. + class CustomFunctions(functions.Functions): + + # 2 and 3. Create a function that starts with _func_ + # and decorate it with @signature which indicates its + # expected types. + # In this example, we're creating a jmespath function + # called "unique_letters" that accepts a single argument + # with an expected type "string". + @functions.signature({'types': ['string']}) + def _func_unique_letters(self, s): + # Given a string s, return a sorted + # string of unique letters: 'ccbbadd' -> 'abcd' + return ''.join(sorted(set(s))) + + # Here's another example. This is creating + # a jmespath function called "my_add" that expects + # two arguments, both of which should be of type number. + @functions.signature({'types': ['number']}, {'types': ['number']}) + def _func_my_add(self, x, y): + return x + y + + # 4. Provide an instance of your subclass in a Options object. + options = jmespath.Options(custom_functions=CustomFunctions()) + + # Provide this value to jmespath.search: + # This will print 3 + print( + jmespath.search( + 'my_add(`1`, `2`)', {}, options=options) + ) + + # This will print "abcd" + print( + jmespath.search( + 'foo.bar | unique_letters(@)', + {'foo': {'bar': 'ccbbadd'}}, + options=options) + ) + +Again, if you come up with useful functions that you think make +sense in the JMESPath language (and make sense to implement in all +JMESPath libraries, not just python), please let us know at +`jmespath.site `__. + + +Specification +============= + +If you'd like to learn more about the JMESPath language, you can check out +the `JMESPath tutorial `__. Also check +out the `JMESPath examples page `__ for +examples of more complex jmespath queries. + +The grammar is specified using ABNF, as described in +`RFC4234 `_. +You can find the most up to date +`grammar for JMESPath here `__. + +You can read the full +`JMESPath specification here `__. + + +Testing +======= + +In addition to the unit tests for the jmespath modules, +there is a ``tests/compliance`` directory that contains +.json files with test cases. This allows other implementations +to verify they are producing the correct output. Each json +file is grouped by feature. + + +Discuss +======= + +Join us on our `Gitter channel `__ +if you want to chat or if you have any questions. + + diff --git a/WebCrawler/venv/Lib/site-packages/jmespath-0.10.0.dist-info/INSTALLER b/WebCrawler/venv/Lib/site-packages/jmespath-0.10.0.dist-info/INSTALLER new file mode 100644 index 0000000..a1b589e --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/jmespath-0.10.0.dist-info/INSTALLER @@ -0,0 +1 @@ +pip diff --git a/WebCrawler/venv/Lib/site-packages/jmespath-0.10.0.dist-info/LICENSE.txt b/WebCrawler/venv/Lib/site-packages/jmespath-0.10.0.dist-info/LICENSE.txt new file mode 100644 index 0000000..aa68928 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/jmespath-0.10.0.dist-info/LICENSE.txt @@ -0,0 +1,20 @@ +Copyright (c) 2013 Amazon.com, Inc. or its affiliates. All Rights Reserved + +Permission is hereby granted, free of charge, to any person obtaining a +copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, dis- +tribute, sublicense, and/or sell copies of the Software, and to permit +persons to whom the Software is furnished to do so, subject to the fol- +lowing conditions: + +The above copyright notice and this permission notice shall be included +in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +IN THE SOFTWARE. diff --git a/WebCrawler/venv/Lib/site-packages/jmespath-0.10.0.dist-info/METADATA b/WebCrawler/venv/Lib/site-packages/jmespath-0.10.0.dist-info/METADATA new file mode 100644 index 0000000..78a9735 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/jmespath-0.10.0.dist-info/METADATA @@ -0,0 +1,251 @@ +Metadata-Version: 2.0 +Name: jmespath +Version: 0.10.0 +Summary: JSON Matching Expressions +Home-page: https://github.com/jmespath/jmespath.py +Author: James Saryerwinnie +Author-email: js@jamesls.com +License: MIT +Platform: UNKNOWN +Classifier: Development Status :: 5 - Production/Stable +Classifier: Intended Audience :: Developers +Classifier: Natural Language :: English +Classifier: License :: OSI Approved :: MIT License +Classifier: Programming Language :: Python +Classifier: Programming Language :: Python :: 2 +Classifier: Programming Language :: Python :: 2.6 +Classifier: Programming Language :: Python :: 2.7 +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3.3 +Classifier: Programming Language :: Python :: 3.4 +Classifier: Programming Language :: Python :: 3.5 +Classifier: Programming Language :: Python :: 3.6 +Classifier: Programming Language :: Python :: 3.7 +Classifier: Programming Language :: Python :: Implementation :: CPython +Classifier: Programming Language :: Python :: Implementation :: PyPy +Requires-Python: >=2.6, !=3.0.*, !=3.1.*, !=3.2.* + +JMESPath +======== + + +.. image:: https://badges.gitter.im/Join Chat.svg + :target: https://gitter.im/jmespath/chat + + +.. image:: https://travis-ci.org/jmespath/jmespath.py.svg?branch=develop + :target: https://travis-ci.org/jmespath/jmespath.py + + +.. image:: https://codecov.io/github/jmespath/jmespath.py/coverage.svg?branch=develop + :target: https://codecov.io/github/jmespath/jmespath.py?branch=develop + + +JMESPath (pronounced "james path") allows you to declaratively specify how to +extract elements from a JSON document. + +For example, given this document:: + + {"foo": {"bar": "baz"}} + +The jmespath expression ``foo.bar`` will return "baz". + +JMESPath also supports: + +Referencing elements in a list. Given the data:: + + {"foo": {"bar": ["one", "two"]}} + +The expression: ``foo.bar[0]`` will return "one". +You can also reference all the items in a list using the ``*`` +syntax:: + + {"foo": {"bar": [{"name": "one"}, {"name": "two"}]}} + +The expression: ``foo.bar[*].name`` will return ["one", "two"]. +Negative indexing is also supported (-1 refers to the last element +in the list). Given the data above, the expression +``foo.bar[-1].name`` will return "two". + +The ``*`` can also be used for hash types:: + + {"foo": {"bar": {"name": "one"}, "baz": {"name": "two"}}} + +The expression: ``foo.*.name`` will return ["one", "two"]. + + +Installation +============ + +You can install JMESPath from pypi with: + +.. code:: bash + + pip install jmespath + + +API +=== + +The ``jmespath.py`` library has two functions +that operate on python data structures. You can use ``search`` +and give it the jmespath expression and the data: + +.. code:: python + + >>> import jmespath + >>> path = jmespath.search('foo.bar', {'foo': {'bar': 'baz'}}) + 'baz' + +Similar to the ``re`` module, you can use the ``compile`` function +to compile the JMESPath expression and use this parsed expression +to perform repeated searches: + +.. code:: python + + >>> import jmespath + >>> expression = jmespath.compile('foo.bar') + >>> expression.search({'foo': {'bar': 'baz'}}) + 'baz' + >>> expression.search({'foo': {'bar': 'other'}}) + 'other' + +This is useful if you're going to use the same jmespath expression to +search multiple documents. This avoids having to reparse the +JMESPath expression each time you search a new document. + +Options +------- + +You can provide an instance of ``jmespath.Options`` to control how +a JMESPath expression is evaluated. The most common scenario for +using an ``Options`` instance is if you want to have ordered output +of your dict keys. To do this you can use either of these options: + +.. code:: python + + >>> import jmespath + >>> jmespath.search('{a: a, b: b}', + ... mydata, + ... jmespath.Options(dict_cls=collections.OrderedDict)) + + + >>> import jmespath + >>> parsed = jmespath.compile('{a: a, b: b}') + >>> parsed.search(mydata, + ... jmespath.Options(dict_cls=collections.OrderedDict)) + + +Custom Functions +~~~~~~~~~~~~~~~~ + +The JMESPath language has numerous +`built-in functions +`__, but it is +also possible to add your own custom functions. Keep in mind that +custom function support in jmespath.py is experimental and the API may +change based on feedback. + +**If you have a custom function that you've found useful, consider submitting +it to jmespath.site and propose that it be added to the JMESPath language.** +You can submit proposals +`here `__. + +To create custom functions: + +* Create a subclass of ``jmespath.functions.Functions``. +* Create a method with the name ``_func_``. +* Apply the ``jmespath.functions.signature`` decorator that indicates + the expected types of the function arguments. +* Provide an instance of your subclass in a ``jmespath.Options`` object. + +Below are a few examples: + +.. code:: python + + import jmespath + from jmespath import functions + + # 1. Create a subclass of functions.Functions. + # The function.Functions base class has logic + # that introspects all of its methods and automatically + # registers your custom functions in its function table. + class CustomFunctions(functions.Functions): + + # 2 and 3. Create a function that starts with _func_ + # and decorate it with @signature which indicates its + # expected types. + # In this example, we're creating a jmespath function + # called "unique_letters" that accepts a single argument + # with an expected type "string". + @functions.signature({'types': ['string']}) + def _func_unique_letters(self, s): + # Given a string s, return a sorted + # string of unique letters: 'ccbbadd' -> 'abcd' + return ''.join(sorted(set(s))) + + # Here's another example. This is creating + # a jmespath function called "my_add" that expects + # two arguments, both of which should be of type number. + @functions.signature({'types': ['number']}, {'types': ['number']}) + def _func_my_add(self, x, y): + return x + y + + # 4. Provide an instance of your subclass in a Options object. + options = jmespath.Options(custom_functions=CustomFunctions()) + + # Provide this value to jmespath.search: + # This will print 3 + print( + jmespath.search( + 'my_add(`1`, `2`)', {}, options=options) + ) + + # This will print "abcd" + print( + jmespath.search( + 'foo.bar | unique_letters(@)', + {'foo': {'bar': 'ccbbadd'}}, + options=options) + ) + +Again, if you come up with useful functions that you think make +sense in the JMESPath language (and make sense to implement in all +JMESPath libraries, not just python), please let us know at +`jmespath.site `__. + + +Specification +============= + +If you'd like to learn more about the JMESPath language, you can check out +the `JMESPath tutorial `__. Also check +out the `JMESPath examples page `__ for +examples of more complex jmespath queries. + +The grammar is specified using ABNF, as described in +`RFC4234 `_. +You can find the most up to date +`grammar for JMESPath here `__. + +You can read the full +`JMESPath specification here `__. + + +Testing +======= + +In addition to the unit tests for the jmespath modules, +there is a ``tests/compliance`` directory that contains +.json files with test cases. This allows other implementations +to verify they are producing the correct output. Each json +file is grouped by feature. + + +Discuss +======= + +Join us on our `Gitter channel `__ +if you want to chat or if you have any questions. + + diff --git a/WebCrawler/venv/Lib/site-packages/jmespath-0.10.0.dist-info/RECORD b/WebCrawler/venv/Lib/site-packages/jmespath-0.10.0.dist-info/RECORD new file mode 100644 index 0000000..3c06815 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/jmespath-0.10.0.dist-info/RECORD @@ -0,0 +1,26 @@ +../../Scripts/__pycache__/jp.cpython-39.pyc,, +../../Scripts/jp.py,sha256=PDO-MtuJ72Ph786r2Aw05wZ5q22GJguev9G0xBXoOI8,1685 +jmespath-0.10.0.dist-info/DESCRIPTION.rst,sha256=pRJREzjsiiQ_PZ72_nUj7-C-cXNkHnJ3iCVCDJpnUyg,6925 +jmespath-0.10.0.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4 +jmespath-0.10.0.dist-info/LICENSE.txt,sha256=ZrMTzOgO0GI_x9s_JIY6DID9g-s0Gka1eGQViudPqlY,1084 +jmespath-0.10.0.dist-info/METADATA,sha256=2JiCaPP_JR87u3ESC8bZLl1-z-2tAx6mHB60DIsGIX0,8023 +jmespath-0.10.0.dist-info/RECORD,, +jmespath-0.10.0.dist-info/WHEEL,sha256=GrqQvamwgBV4nLoJe0vhYRSWzWsx7xjlt74FT0SWYfE,110 +jmespath-0.10.0.dist-info/metadata.json,sha256=Zyl6gtawhruY1SbwtILEz8xueg-HiojsyRsudgAudBs,1188 +jmespath-0.10.0.dist-info/top_level.txt,sha256=vuy_oZ1ckpeSNrAi8JK8-yIGO6bduO3qvW2cCMQmPH8,9 +jmespath/__init__.py,sha256=3cgzWR1zfRQaIs83GD33B1LDw6AzqCTEhY_2s5JMUI8,623 +jmespath/__pycache__/__init__.cpython-39.pyc,, +jmespath/__pycache__/ast.cpython-39.pyc,, +jmespath/__pycache__/compat.cpython-39.pyc,, +jmespath/__pycache__/exceptions.cpython-39.pyc,, +jmespath/__pycache__/functions.cpython-39.pyc,, +jmespath/__pycache__/lexer.cpython-39.pyc,, +jmespath/__pycache__/parser.cpython-39.pyc,, +jmespath/__pycache__/visitor.cpython-39.pyc,, +jmespath/ast.py,sha256=SiHRM1mdQPRt12R2qkQu_Ezbd-ghk5HLy1Hl1jHAyv8,2130 +jmespath/compat.py,sha256=Q_a-dEOGrKa9Mimrhfpuq5r-9poyBz9ZDvbzSpPE0Tw,2117 +jmespath/exceptions.py,sha256=5HRot4Bv3SCTdvh5lAQdQBsqSdaKjuTPSyGmk08zSvA,4128 +jmespath/functions.py,sha256=1ZwvCNVzHU1hFUUeNbP9LO_v_GqJPAGH-xxF98vNen8,12766 +jmespath/lexer.py,sha256=LE3-dM75ny9RMR8mXmWlKKRTIlz8njvWz-M-2y4t5WA,8574 +jmespath/parser.py,sha256=XwLffptCw5f3PKoJYI7C_pbAX5MKzxFECwnVqD7fk3A,19082 +jmespath/visitor.py,sha256=jSGILEeuY16WkMkxel2yytcbljVNtNtLW4Bx0qfV4cU,10844 diff --git a/WebCrawler/venv/Lib/site-packages/jmespath-0.10.0.dist-info/WHEEL b/WebCrawler/venv/Lib/site-packages/jmespath-0.10.0.dist-info/WHEEL new file mode 100644 index 0000000..0de529b --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/jmespath-0.10.0.dist-info/WHEEL @@ -0,0 +1,6 @@ +Wheel-Version: 1.0 +Generator: bdist_wheel (0.26.0) +Root-Is-Purelib: true +Tag: py2-none-any +Tag: py3-none-any + diff --git a/WebCrawler/venv/Lib/site-packages/jmespath-0.10.0.dist-info/metadata.json b/WebCrawler/venv/Lib/site-packages/jmespath-0.10.0.dist-info/metadata.json new file mode 100644 index 0000000..dbf1f68 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/jmespath-0.10.0.dist-info/metadata.json @@ -0,0 +1 @@ +{"generator": "bdist_wheel (0.26.0)", "summary": "JSON Matching Expressions", "classifiers": ["Development Status :: 5 - Production/Stable", "Intended Audience :: Developers", "Natural Language :: English", "License :: OSI Approved :: MIT License", "Programming Language :: Python", "Programming Language :: Python :: 2", "Programming Language :: Python :: 2.6", "Programming Language :: Python :: 2.7", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.3", "Programming Language :: Python :: 3.4", "Programming Language :: Python :: 3.5", "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy"], "extensions": {"python.details": {"project_urls": {"Home": "https://github.com/jmespath/jmespath.py"}, "contacts": [{"email": "js@jamesls.com", "name": "James Saryerwinnie", "role": "author"}], "document_names": {"description": "DESCRIPTION.rst", "license": "LICENSE.txt"}}}, "license": "MIT", "metadata_version": "2.0", "name": "jmespath", "requires_python": ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*", "version": "0.10.0"} \ No newline at end of file diff --git a/WebCrawler/venv/Lib/site-packages/jmespath-0.10.0.dist-info/top_level.txt b/WebCrawler/venv/Lib/site-packages/jmespath-0.10.0.dist-info/top_level.txt new file mode 100644 index 0000000..45c1e03 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/jmespath-0.10.0.dist-info/top_level.txt @@ -0,0 +1 @@ +jmespath diff --git a/WebCrawler/venv/Lib/site-packages/jmespath/__init__.py b/WebCrawler/venv/Lib/site-packages/jmespath/__init__.py new file mode 100644 index 0000000..99482db --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/jmespath/__init__.py @@ -0,0 +1,23 @@ +import warnings +import sys +from jmespath import parser +from jmespath.visitor import Options + +__version__ = '0.10.0' + + +if sys.version_info[:2] <= (2, 6) or ((3, 0) <= sys.version_info[:2] <= (3, 3)): + python_ver = '.'.join(str(x) for x in sys.version_info[:3]) + + warnings.warn( + 'You are using Python {0}, which will no longer be supported in ' + 'version 0.11.0'.format(python_ver), + DeprecationWarning) + + +def compile(expression): + return parser.Parser().parse(expression) + + +def search(expression, data, options=None): + return parser.Parser().parse(expression).search(data, options=options) diff --git a/WebCrawler/venv/Lib/site-packages/jmespath/__pycache__/__init__.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/jmespath/__pycache__/__init__.cpython-39.pyc new file mode 100644 index 0000000..42a41ff Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/jmespath/__pycache__/__init__.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/jmespath/__pycache__/ast.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/jmespath/__pycache__/ast.cpython-39.pyc new file mode 100644 index 0000000..cc84c2b Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/jmespath/__pycache__/ast.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/jmespath/__pycache__/compat.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/jmespath/__pycache__/compat.cpython-39.pyc new file mode 100644 index 0000000..277e46e Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/jmespath/__pycache__/compat.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/jmespath/__pycache__/exceptions.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/jmespath/__pycache__/exceptions.cpython-39.pyc new file mode 100644 index 0000000..7cce59e Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/jmespath/__pycache__/exceptions.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/jmespath/__pycache__/functions.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/jmespath/__pycache__/functions.cpython-39.pyc new file mode 100644 index 0000000..d5dd7e2 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/jmespath/__pycache__/functions.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/jmespath/__pycache__/lexer.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/jmespath/__pycache__/lexer.cpython-39.pyc new file mode 100644 index 0000000..3be4077 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/jmespath/__pycache__/lexer.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/jmespath/__pycache__/parser.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/jmespath/__pycache__/parser.cpython-39.pyc new file mode 100644 index 0000000..a0a5847 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/jmespath/__pycache__/parser.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/jmespath/__pycache__/visitor.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/jmespath/__pycache__/visitor.cpython-39.pyc new file mode 100644 index 0000000..04c9ca5 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/jmespath/__pycache__/visitor.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/jmespath/ast.py b/WebCrawler/venv/Lib/site-packages/jmespath/ast.py new file mode 100644 index 0000000..dd56c6e --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/jmespath/ast.py @@ -0,0 +1,90 @@ +# AST nodes have this structure: +# {"type": ", children: [], "value": ""} + + +def comparator(name, first, second): + return {'type': 'comparator', 'children': [first, second], 'value': name} + + +def current_node(): + return {'type': 'current', 'children': []} + + +def expref(expression): + return {'type': 'expref', 'children': [expression]} + + +def function_expression(name, args): + return {'type': 'function_expression', 'children': args, 'value': name} + + +def field(name): + return {"type": "field", "children": [], "value": name} + + +def filter_projection(left, right, comparator): + return {'type': 'filter_projection', 'children': [left, right, comparator]} + + +def flatten(node): + return {'type': 'flatten', 'children': [node]} + + +def identity(): + return {"type": "identity", 'children': []} + + +def index(index): + return {"type": "index", "value": index, "children": []} + + +def index_expression(children): + return {"type": "index_expression", 'children': children} + + +def key_val_pair(key_name, node): + return {"type": "key_val_pair", 'children': [node], "value": key_name} + + +def literal(literal_value): + return {'type': 'literal', 'value': literal_value, 'children': []} + + +def multi_select_dict(nodes): + return {"type": "multi_select_dict", "children": nodes} + + +def multi_select_list(nodes): + return {"type": "multi_select_list", "children": nodes} + + +def or_expression(left, right): + return {"type": "or_expression", "children": [left, right]} + + +def and_expression(left, right): + return {"type": "and_expression", "children": [left, right]} + + +def not_expression(expr): + return {"type": "not_expression", "children": [expr]} + + +def pipe(left, right): + return {'type': 'pipe', 'children': [left, right]} + + +def projection(left, right): + return {'type': 'projection', 'children': [left, right]} + + +def subexpression(children): + return {"type": "subexpression", 'children': children} + + +def slice(start, end, step): + return {"type": "slice", "children": [start, end, step]} + + +def value_projection(left, right): + return {'type': 'value_projection', 'children': [left, right]} diff --git a/WebCrawler/venv/Lib/site-packages/jmespath/compat.py b/WebCrawler/venv/Lib/site-packages/jmespath/compat.py new file mode 100644 index 0000000..2ed0fe7 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/jmespath/compat.py @@ -0,0 +1,65 @@ +import sys +import inspect + +PY2 = sys.version_info[0] == 2 + + +def with_metaclass(meta, *bases): + # Taken from flask/six. + class metaclass(meta): + def __new__(cls, name, this_bases, d): + return meta(name, bases, d) + return type.__new__(metaclass, 'temporary_class', (), {}) + + +if PY2: + text_type = unicode + string_type = basestring + from itertools import izip_longest as zip_longest + + def with_str_method(cls): + """Class decorator that handles __str__ compat between py2 and py3.""" + # In python2, the __str__ should be __unicode__ + # and __str__ should return bytes. + cls.__unicode__ = cls.__str__ + def __str__(self): + return self.__unicode__().encode('utf-8') + cls.__str__ = __str__ + return cls + + def with_repr_method(cls): + """Class decorator that handle __repr__ with py2 and py3.""" + # This is almost the same thing as with_str_method *except* + # it uses the unicode_escape encoding. This also means we need to be + # careful encoding the input multiple times, so we only encode + # if we get a unicode type. + original_repr_method = cls.__repr__ + def __repr__(self): + original_repr = original_repr_method(self) + if isinstance(original_repr, text_type): + original_repr = original_repr.encode('unicode_escape') + return original_repr + cls.__repr__ = __repr__ + return cls + + def get_methods(cls): + for name, method in inspect.getmembers(cls, + predicate=inspect.ismethod): + yield name, method + +else: + text_type = str + string_type = str + from itertools import zip_longest + + def with_str_method(cls): + # In python3, we don't need to do anything, we return a str type. + return cls + + def with_repr_method(cls): + return cls + + def get_methods(cls): + for name, method in inspect.getmembers(cls, + predicate=inspect.isfunction): + yield name, method diff --git a/WebCrawler/venv/Lib/site-packages/jmespath/exceptions.py b/WebCrawler/venv/Lib/site-packages/jmespath/exceptions.py new file mode 100644 index 0000000..0156015 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/jmespath/exceptions.py @@ -0,0 +1,122 @@ +from jmespath.compat import with_str_method + + +class JMESPathError(ValueError): + pass + + +@with_str_method +class ParseError(JMESPathError): + _ERROR_MESSAGE = 'Invalid jmespath expression' + def __init__(self, lex_position, token_value, token_type, + msg=_ERROR_MESSAGE): + super(ParseError, self).__init__(lex_position, token_value, token_type) + self.lex_position = lex_position + self.token_value = token_value + self.token_type = token_type.upper() + self.msg = msg + # Whatever catches the ParseError can fill in the full expression + self.expression = None + + def __str__(self): + # self.lex_position +1 to account for the starting double quote char. + underline = ' ' * (self.lex_position + 1) + '^' + return ( + '%s: Parse error at column %s, ' + 'token "%s" (%s), for expression:\n"%s"\n%s' % ( + self.msg, self.lex_position, self.token_value, self.token_type, + self.expression, underline)) + + +@with_str_method +class IncompleteExpressionError(ParseError): + def set_expression(self, expression): + self.expression = expression + self.lex_position = len(expression) + self.token_type = None + self.token_value = None + + def __str__(self): + # self.lex_position +1 to account for the starting double quote char. + underline = ' ' * (self.lex_position + 1) + '^' + return ( + 'Invalid jmespath expression: Incomplete expression:\n' + '"%s"\n%s' % (self.expression, underline)) + + +@with_str_method +class LexerError(ParseError): + def __init__(self, lexer_position, lexer_value, message, expression=None): + self.lexer_position = lexer_position + self.lexer_value = lexer_value + self.message = message + super(LexerError, self).__init__(lexer_position, + lexer_value, + message) + # Whatever catches LexerError can set this. + self.expression = expression + + def __str__(self): + underline = ' ' * self.lexer_position + '^' + return 'Bad jmespath expression: %s:\n%s\n%s' % ( + self.message, self.expression, underline) + + +@with_str_method +class ArityError(ParseError): + def __init__(self, expected, actual, name): + self.expected_arity = expected + self.actual_arity = actual + self.function_name = name + self.expression = None + + def __str__(self): + return ("Expected %s %s for function %s(), " + "received %s" % ( + self.expected_arity, + self._pluralize('argument', self.expected_arity), + self.function_name, + self.actual_arity)) + + def _pluralize(self, word, count): + if count == 1: + return word + else: + return word + 's' + + +@with_str_method +class VariadictArityError(ArityError): + def __str__(self): + return ("Expected at least %s %s for function %s(), " + "received %s" % ( + self.expected_arity, + self._pluralize('argument', self.expected_arity), + self.function_name, + self.actual_arity)) + + +@with_str_method +class JMESPathTypeError(JMESPathError): + def __init__(self, function_name, current_value, actual_type, + expected_types): + self.function_name = function_name + self.current_value = current_value + self.actual_type = actual_type + self.expected_types = expected_types + + def __str__(self): + return ('In function %s(), invalid type for value: %s, ' + 'expected one of: %s, received: "%s"' % ( + self.function_name, self.current_value, + self.expected_types, self.actual_type)) + + +class EmptyExpressionError(JMESPathError): + def __init__(self): + super(EmptyExpressionError, self).__init__( + "Invalid JMESPath expression: cannot be empty.") + + +class UnknownFunctionError(JMESPathError): + pass diff --git a/WebCrawler/venv/Lib/site-packages/jmespath/functions.py b/WebCrawler/venv/Lib/site-packages/jmespath/functions.py new file mode 100644 index 0000000..31dab05 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/jmespath/functions.py @@ -0,0 +1,362 @@ +import math +import json + +from jmespath import exceptions +from jmespath.compat import string_type as STRING_TYPE +from jmespath.compat import get_methods, with_metaclass + + +# python types -> jmespath types +TYPES_MAP = { + 'bool': 'boolean', + 'list': 'array', + 'dict': 'object', + 'NoneType': 'null', + 'unicode': 'string', + 'str': 'string', + 'float': 'number', + 'int': 'number', + 'long': 'number', + 'OrderedDict': 'object', + '_Projection': 'array', + '_Expression': 'expref', +} + + +# jmespath types -> python types +REVERSE_TYPES_MAP = { + 'boolean': ('bool',), + 'array': ('list', '_Projection'), + 'object': ('dict', 'OrderedDict',), + 'null': ('NoneType',), + 'string': ('unicode', 'str'), + 'number': ('float', 'int', 'long'), + 'expref': ('_Expression',), +} + + +def signature(*arguments): + def _record_signature(func): + func.signature = arguments + return func + return _record_signature + + +class FunctionRegistry(type): + def __init__(cls, name, bases, attrs): + cls._populate_function_table() + super(FunctionRegistry, cls).__init__(name, bases, attrs) + + def _populate_function_table(cls): + function_table = {} + # Any method with a @signature decorator that also + # starts with "_func_" is registered as a function. + # _func_max_by -> max_by function. + for name, method in get_methods(cls): + if not name.startswith('_func_'): + continue + signature = getattr(method, 'signature', None) + if signature is not None: + function_table[name[6:]] = { + 'function': method, + 'signature': signature, + } + cls.FUNCTION_TABLE = function_table + + +class Functions(with_metaclass(FunctionRegistry, object)): + + FUNCTION_TABLE = { + } + + def call_function(self, function_name, resolved_args): + try: + spec = self.FUNCTION_TABLE[function_name] + except KeyError: + raise exceptions.UnknownFunctionError( + "Unknown function: %s()" % function_name) + function = spec['function'] + signature = spec['signature'] + self._validate_arguments(resolved_args, signature, function_name) + return function(self, *resolved_args) + + def _validate_arguments(self, args, signature, function_name): + if signature and signature[-1].get('variadic'): + if len(args) < len(signature): + raise exceptions.VariadictArityError( + len(signature), len(args), function_name) + elif len(args) != len(signature): + raise exceptions.ArityError( + len(signature), len(args), function_name) + return self._type_check(args, signature, function_name) + + def _type_check(self, actual, signature, function_name): + for i in range(len(signature)): + allowed_types = signature[i]['types'] + if allowed_types: + self._type_check_single(actual[i], allowed_types, + function_name) + + def _type_check_single(self, current, types, function_name): + # Type checking involves checking the top level type, + # and in the case of arrays, potentially checking the types + # of each element. + allowed_types, allowed_subtypes = self._get_allowed_pytypes(types) + # We're not using isinstance() on purpose. + # The type model for jmespath does not map + # 1-1 with python types (booleans are considered + # integers in python for example). + actual_typename = type(current).__name__ + if actual_typename not in allowed_types: + raise exceptions.JMESPathTypeError( + function_name, current, + self._convert_to_jmespath_type(actual_typename), types) + # If we're dealing with a list type, we can have + # additional restrictions on the type of the list + # elements (for example a function can require a + # list of numbers or a list of strings). + # Arrays are the only types that can have subtypes. + if allowed_subtypes: + self._subtype_check(current, allowed_subtypes, + types, function_name) + + def _get_allowed_pytypes(self, types): + allowed_types = [] + allowed_subtypes = [] + for t in types: + type_ = t.split('-', 1) + if len(type_) == 2: + type_, subtype = type_ + allowed_subtypes.append(REVERSE_TYPES_MAP[subtype]) + else: + type_ = type_[0] + allowed_types.extend(REVERSE_TYPES_MAP[type_]) + return allowed_types, allowed_subtypes + + def _subtype_check(self, current, allowed_subtypes, types, function_name): + if len(allowed_subtypes) == 1: + # The easy case, we know up front what type + # we need to validate. + allowed_subtypes = allowed_subtypes[0] + for element in current: + actual_typename = type(element).__name__ + if actual_typename not in allowed_subtypes: + raise exceptions.JMESPathTypeError( + function_name, element, actual_typename, types) + elif len(allowed_subtypes) > 1 and current: + # Dynamic type validation. Based on the first + # type we see, we validate that the remaining types + # match. + first = type(current[0]).__name__ + for subtypes in allowed_subtypes: + if first in subtypes: + allowed = subtypes + break + else: + raise exceptions.JMESPathTypeError( + function_name, current[0], first, types) + for element in current: + actual_typename = type(element).__name__ + if actual_typename not in allowed: + raise exceptions.JMESPathTypeError( + function_name, element, actual_typename, types) + + @signature({'types': ['number']}) + def _func_abs(self, arg): + return abs(arg) + + @signature({'types': ['array-number']}) + def _func_avg(self, arg): + if arg: + return sum(arg) / float(len(arg)) + else: + return None + + @signature({'types': [], 'variadic': True}) + def _func_not_null(self, *arguments): + for argument in arguments: + if argument is not None: + return argument + + @signature({'types': []}) + def _func_to_array(self, arg): + if isinstance(arg, list): + return arg + else: + return [arg] + + @signature({'types': []}) + def _func_to_string(self, arg): + if isinstance(arg, STRING_TYPE): + return arg + else: + return json.dumps(arg, separators=(',', ':'), + default=str) + + @signature({'types': []}) + def _func_to_number(self, arg): + if isinstance(arg, (list, dict, bool)): + return None + elif arg is None: + return None + elif isinstance(arg, (int, float)): + return arg + else: + try: + return int(arg) + except ValueError: + try: + return float(arg) + except ValueError: + return None + + @signature({'types': ['array', 'string']}, {'types': []}) + def _func_contains(self, subject, search): + return search in subject + + @signature({'types': ['string', 'array', 'object']}) + def _func_length(self, arg): + return len(arg) + + @signature({'types': ['string']}, {'types': ['string']}) + def _func_ends_with(self, search, suffix): + return search.endswith(suffix) + + @signature({'types': ['string']}, {'types': ['string']}) + def _func_starts_with(self, search, suffix): + return search.startswith(suffix) + + @signature({'types': ['array', 'string']}) + def _func_reverse(self, arg): + if isinstance(arg, STRING_TYPE): + return arg[::-1] + else: + return list(reversed(arg)) + + @signature({"types": ['number']}) + def _func_ceil(self, arg): + return math.ceil(arg) + + @signature({"types": ['number']}) + def _func_floor(self, arg): + return math.floor(arg) + + @signature({"types": ['string']}, {"types": ['array-string']}) + def _func_join(self, separator, array): + return separator.join(array) + + @signature({'types': ['expref']}, {'types': ['array']}) + def _func_map(self, expref, arg): + result = [] + for element in arg: + result.append(expref.visit(expref.expression, element)) + return result + + @signature({"types": ['array-number', 'array-string']}) + def _func_max(self, arg): + if arg: + return max(arg) + else: + return None + + @signature({"types": ["object"], "variadic": True}) + def _func_merge(self, *arguments): + merged = {} + for arg in arguments: + merged.update(arg) + return merged + + @signature({"types": ['array-number', 'array-string']}) + def _func_min(self, arg): + if arg: + return min(arg) + else: + return None + + @signature({"types": ['array-string', 'array-number']}) + def _func_sort(self, arg): + return list(sorted(arg)) + + @signature({"types": ['array-number']}) + def _func_sum(self, arg): + return sum(arg) + + @signature({"types": ['object']}) + def _func_keys(self, arg): + # To be consistent with .values() + # should we also return the indices of a list? + return list(arg.keys()) + + @signature({"types": ['object']}) + def _func_values(self, arg): + return list(arg.values()) + + @signature({'types': []}) + def _func_type(self, arg): + if isinstance(arg, STRING_TYPE): + return "string" + elif isinstance(arg, bool): + return "boolean" + elif isinstance(arg, list): + return "array" + elif isinstance(arg, dict): + return "object" + elif isinstance(arg, (float, int)): + return "number" + elif arg is None: + return "null" + + @signature({'types': ['array']}, {'types': ['expref']}) + def _func_sort_by(self, array, expref): + if not array: + return array + # sort_by allows for the expref to be either a number of + # a string, so we have some special logic to handle this. + # We evaluate the first array element and verify that it's + # either a string of a number. We then create a key function + # that validates that type, which requires that remaining array + # elements resolve to the same type as the first element. + required_type = self._convert_to_jmespath_type( + type(expref.visit(expref.expression, array[0])).__name__) + if required_type not in ['number', 'string']: + raise exceptions.JMESPathTypeError( + 'sort_by', array[0], required_type, ['string', 'number']) + keyfunc = self._create_key_func(expref, + [required_type], + 'sort_by') + return list(sorted(array, key=keyfunc)) + + @signature({'types': ['array']}, {'types': ['expref']}) + def _func_min_by(self, array, expref): + keyfunc = self._create_key_func(expref, + ['number', 'string'], + 'min_by') + if array: + return min(array, key=keyfunc) + else: + return None + + @signature({'types': ['array']}, {'types': ['expref']}) + def _func_max_by(self, array, expref): + keyfunc = self._create_key_func(expref, + ['number', 'string'], + 'max_by') + if array: + return max(array, key=keyfunc) + else: + return None + + def _create_key_func(self, expref, allowed_types, function_name): + def keyfunc(x): + result = expref.visit(expref.expression, x) + actual_typename = type(result).__name__ + jmespath_type = self._convert_to_jmespath_type(actual_typename) + # allowed_types is in term of jmespath types, not python types. + if jmespath_type not in allowed_types: + raise exceptions.JMESPathTypeError( + function_name, result, jmespath_type, allowed_types) + return result + return keyfunc + + def _convert_to_jmespath_type(self, pyobject): + return TYPES_MAP.get(pyobject, 'unknown') diff --git a/WebCrawler/venv/Lib/site-packages/jmespath/lexer.py b/WebCrawler/venv/Lib/site-packages/jmespath/lexer.py new file mode 100644 index 0000000..8db05e3 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/jmespath/lexer.py @@ -0,0 +1,208 @@ +import string +import warnings +from json import loads + +from jmespath.exceptions import LexerError, EmptyExpressionError + + +class Lexer(object): + START_IDENTIFIER = set(string.ascii_letters + '_') + VALID_IDENTIFIER = set(string.ascii_letters + string.digits + '_') + VALID_NUMBER = set(string.digits) + WHITESPACE = set(" \t\n\r") + SIMPLE_TOKENS = { + '.': 'dot', + '*': 'star', + ']': 'rbracket', + ',': 'comma', + ':': 'colon', + '@': 'current', + '(': 'lparen', + ')': 'rparen', + '{': 'lbrace', + '}': 'rbrace', + } + + def tokenize(self, expression): + self._initialize_for_expression(expression) + while self._current is not None: + if self._current in self.SIMPLE_TOKENS: + yield {'type': self.SIMPLE_TOKENS[self._current], + 'value': self._current, + 'start': self._position, 'end': self._position + 1} + self._next() + elif self._current in self.START_IDENTIFIER: + start = self._position + buff = self._current + while self._next() in self.VALID_IDENTIFIER: + buff += self._current + yield {'type': 'unquoted_identifier', 'value': buff, + 'start': start, 'end': start + len(buff)} + elif self._current in self.WHITESPACE: + self._next() + elif self._current == '[': + start = self._position + next_char = self._next() + if next_char == ']': + self._next() + yield {'type': 'flatten', 'value': '[]', + 'start': start, 'end': start + 2} + elif next_char == '?': + self._next() + yield {'type': 'filter', 'value': '[?', + 'start': start, 'end': start + 2} + else: + yield {'type': 'lbracket', 'value': '[', + 'start': start, 'end': start + 1} + elif self._current == "'": + yield self._consume_raw_string_literal() + elif self._current == '|': + yield self._match_or_else('|', 'or', 'pipe') + elif self._current == '&': + yield self._match_or_else('&', 'and', 'expref') + elif self._current == '`': + yield self._consume_literal() + elif self._current in self.VALID_NUMBER: + start = self._position + buff = self._consume_number() + yield {'type': 'number', 'value': int(buff), + 'start': start, 'end': start + len(buff)} + elif self._current == '-': + # Negative number. + start = self._position + buff = self._consume_number() + if len(buff) > 1: + yield {'type': 'number', 'value': int(buff), + 'start': start, 'end': start + len(buff)} + else: + raise LexerError(lexer_position=start, + lexer_value=buff, + message="Unknown token '%s'" % buff) + elif self._current == '"': + yield self._consume_quoted_identifier() + elif self._current == '<': + yield self._match_or_else('=', 'lte', 'lt') + elif self._current == '>': + yield self._match_or_else('=', 'gte', 'gt') + elif self._current == '!': + yield self._match_or_else('=', 'ne', 'not') + elif self._current == '=': + if self._next() == '=': + yield {'type': 'eq', 'value': '==', + 'start': self._position - 1, 'end': self._position} + self._next() + else: + if self._current is None: + # If we're at the EOF, we never advanced + # the position so we don't need to rewind + # it back one location. + position = self._position + else: + position = self._position - 1 + raise LexerError( + lexer_position=position, + lexer_value='=', + message="Unknown token '='") + else: + raise LexerError(lexer_position=self._position, + lexer_value=self._current, + message="Unknown token %s" % self._current) + yield {'type': 'eof', 'value': '', + 'start': self._length, 'end': self._length} + + def _consume_number(self): + start = self._position + buff = self._current + while self._next() in self.VALID_NUMBER: + buff += self._current + return buff + + def _initialize_for_expression(self, expression): + if not expression: + raise EmptyExpressionError() + self._position = 0 + self._expression = expression + self._chars = list(self._expression) + self._current = self._chars[self._position] + self._length = len(self._expression) + + def _next(self): + if self._position == self._length - 1: + self._current = None + else: + self._position += 1 + self._current = self._chars[self._position] + return self._current + + def _consume_until(self, delimiter): + # Consume until the delimiter is reached, + # allowing for the delimiter to be escaped with "\". + start = self._position + buff = '' + self._next() + while self._current != delimiter: + if self._current == '\\': + buff += '\\' + self._next() + if self._current is None: + # We're at the EOF. + raise LexerError(lexer_position=start, + lexer_value=self._expression[start:], + message="Unclosed %s delimiter" % delimiter) + buff += self._current + self._next() + # Skip the closing delimiter. + self._next() + return buff + + def _consume_literal(self): + start = self._position + lexeme = self._consume_until('`').replace('\\`', '`') + try: + # Assume it is valid JSON and attempt to parse. + parsed_json = loads(lexeme) + except ValueError: + try: + # Invalid JSON values should be converted to quoted + # JSON strings during the JEP-12 deprecation period. + parsed_json = loads('"%s"' % lexeme.lstrip()) + warnings.warn("deprecated string literal syntax", + PendingDeprecationWarning) + except ValueError: + raise LexerError(lexer_position=start, + lexer_value=self._expression[start:], + message="Bad token %s" % lexeme) + token_len = self._position - start + return {'type': 'literal', 'value': parsed_json, + 'start': start, 'end': token_len} + + def _consume_quoted_identifier(self): + start = self._position + lexeme = '"' + self._consume_until('"') + '"' + try: + token_len = self._position - start + return {'type': 'quoted_identifier', 'value': loads(lexeme), + 'start': start, 'end': token_len} + except ValueError as e: + error_message = str(e).split(':')[0] + raise LexerError(lexer_position=start, + lexer_value=lexeme, + message=error_message) + + def _consume_raw_string_literal(self): + start = self._position + lexeme = self._consume_until("'").replace("\\'", "'") + token_len = self._position - start + return {'type': 'literal', 'value': lexeme, + 'start': start, 'end': token_len} + + def _match_or_else(self, expected, match_type, else_type): + start = self._position + current = self._current + next_char = self._next() + if next_char == expected: + self._next() + return {'type': match_type, 'value': current + next_char, + 'start': start, 'end': start + 1} + return {'type': else_type, 'value': current, + 'start': start, 'end': start} diff --git a/WebCrawler/venv/Lib/site-packages/jmespath/parser.py b/WebCrawler/venv/Lib/site-packages/jmespath/parser.py new file mode 100644 index 0000000..eeac38f --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/jmespath/parser.py @@ -0,0 +1,527 @@ +"""Top down operator precedence parser. + +This is an implementation of Vaughan R. Pratt's +"Top Down Operator Precedence" parser. +(http://dl.acm.org/citation.cfm?doid=512927.512931). + +These are some additional resources that help explain the +general idea behind a Pratt parser: + +* http://effbot.org/zone/simple-top-down-parsing.htm +* http://javascript.crockford.com/tdop/tdop.html + +A few notes on the implementation. + +* All the nud/led tokens are on the Parser class itself, and are dispatched + using getattr(). This keeps all the parsing logic contained to a single + class. +* We use two passes through the data. One to create a list of token, + then one pass through the tokens to create the AST. While the lexer actually + yields tokens, we convert it to a list so we can easily implement two tokens + of lookahead. A previous implementation used a fixed circular buffer, but it + was significantly slower. Also, the average jmespath expression typically + does not have a large amount of token so this is not an issue. And + interestingly enough, creating a token list first is actually faster than + consuming from the token iterator one token at a time. + +""" +import random + +from jmespath import lexer +from jmespath.compat import with_repr_method +from jmespath import ast +from jmespath import exceptions +from jmespath import visitor + + +class Parser(object): + BINDING_POWER = { + 'eof': 0, + 'unquoted_identifier': 0, + 'quoted_identifier': 0, + 'literal': 0, + 'rbracket': 0, + 'rparen': 0, + 'comma': 0, + 'rbrace': 0, + 'number': 0, + 'current': 0, + 'expref': 0, + 'colon': 0, + 'pipe': 1, + 'or': 2, + 'and': 3, + 'eq': 5, + 'gt': 5, + 'lt': 5, + 'gte': 5, + 'lte': 5, + 'ne': 5, + 'flatten': 9, + # Everything above stops a projection. + 'star': 20, + 'filter': 21, + 'dot': 40, + 'not': 45, + 'lbrace': 50, + 'lbracket': 55, + 'lparen': 60, + } + # The maximum binding power for a token that can stop + # a projection. + _PROJECTION_STOP = 10 + # The _MAX_SIZE most recent expressions are cached in + # _CACHE dict. + _CACHE = {} + _MAX_SIZE = 128 + + def __init__(self, lookahead=2): + self.tokenizer = None + self._tokens = [None] * lookahead + self._buffer_size = lookahead + self._index = 0 + + def parse(self, expression): + cached = self._CACHE.get(expression) + if cached is not None: + return cached + parsed_result = self._do_parse(expression) + self._CACHE[expression] = parsed_result + if len(self._CACHE) > self._MAX_SIZE: + self._free_cache_entries() + return parsed_result + + def _do_parse(self, expression): + try: + return self._parse(expression) + except exceptions.LexerError as e: + e.expression = expression + raise + except exceptions.IncompleteExpressionError as e: + e.set_expression(expression) + raise + except exceptions.ParseError as e: + e.expression = expression + raise + + def _parse(self, expression): + self.tokenizer = lexer.Lexer().tokenize(expression) + self._tokens = list(self.tokenizer) + self._index = 0 + parsed = self._expression(binding_power=0) + if not self._current_token() == 'eof': + t = self._lookahead_token(0) + raise exceptions.ParseError(t['start'], t['value'], t['type'], + "Unexpected token: %s" % t['value']) + return ParsedResult(expression, parsed) + + def _expression(self, binding_power=0): + left_token = self._lookahead_token(0) + self._advance() + nud_function = getattr( + self, '_token_nud_%s' % left_token['type'], + self._error_nud_token) + left = nud_function(left_token) + current_token = self._current_token() + while binding_power < self.BINDING_POWER[current_token]: + led = getattr(self, '_token_led_%s' % current_token, None) + if led is None: + error_token = self._lookahead_token(0) + self._error_led_token(error_token) + else: + self._advance() + left = led(left) + current_token = self._current_token() + return left + + def _token_nud_literal(self, token): + return ast.literal(token['value']) + + def _token_nud_unquoted_identifier(self, token): + return ast.field(token['value']) + + def _token_nud_quoted_identifier(self, token): + field = ast.field(token['value']) + # You can't have a quoted identifier as a function + # name. + if self._current_token() == 'lparen': + t = self._lookahead_token(0) + raise exceptions.ParseError( + 0, t['value'], t['type'], + 'Quoted identifier not allowed for function names.') + return field + + def _token_nud_star(self, token): + left = ast.identity() + if self._current_token() == 'rbracket': + right = ast.identity() + else: + right = self._parse_projection_rhs(self.BINDING_POWER['star']) + return ast.value_projection(left, right) + + def _token_nud_filter(self, token): + return self._token_led_filter(ast.identity()) + + def _token_nud_lbrace(self, token): + return self._parse_multi_select_hash() + + def _token_nud_lparen(self, token): + expression = self._expression() + self._match('rparen') + return expression + + def _token_nud_flatten(self, token): + left = ast.flatten(ast.identity()) + right = self._parse_projection_rhs( + self.BINDING_POWER['flatten']) + return ast.projection(left, right) + + def _token_nud_not(self, token): + expr = self._expression(self.BINDING_POWER['not']) + return ast.not_expression(expr) + + def _token_nud_lbracket(self, token): + if self._current_token() in ['number', 'colon']: + right = self._parse_index_expression() + # We could optimize this and remove the identity() node. + # We don't really need an index_expression node, we can + # just use emit an index node here if we're not dealing + # with a slice. + return self._project_if_slice(ast.identity(), right) + elif self._current_token() == 'star' and \ + self._lookahead(1) == 'rbracket': + self._advance() + self._advance() + right = self._parse_projection_rhs(self.BINDING_POWER['star']) + return ast.projection(ast.identity(), right) + else: + return self._parse_multi_select_list() + + def _parse_index_expression(self): + # We're here: + # [ + # ^ + # | current token + if (self._lookahead(0) == 'colon' or + self._lookahead(1) == 'colon'): + return self._parse_slice_expression() + else: + # Parse the syntax [number] + node = ast.index(self._lookahead_token(0)['value']) + self._advance() + self._match('rbracket') + return node + + def _parse_slice_expression(self): + # [start:end:step] + # Where start, end, and step are optional. + # The last colon is optional as well. + parts = [None, None, None] + index = 0 + current_token = self._current_token() + while not current_token == 'rbracket' and index < 3: + if current_token == 'colon': + index += 1 + if index == 3: + self._raise_parse_error_for_token( + self._lookahead_token(0), 'syntax error') + self._advance() + elif current_token == 'number': + parts[index] = self._lookahead_token(0)['value'] + self._advance() + else: + self._raise_parse_error_for_token( + self._lookahead_token(0), 'syntax error') + current_token = self._current_token() + self._match('rbracket') + return ast.slice(*parts) + + def _token_nud_current(self, token): + return ast.current_node() + + def _token_nud_expref(self, token): + expression = self._expression(self.BINDING_POWER['expref']) + return ast.expref(expression) + + def _token_led_dot(self, left): + if not self._current_token() == 'star': + right = self._parse_dot_rhs(self.BINDING_POWER['dot']) + if left['type'] == 'subexpression': + left['children'].append(right) + return left + else: + return ast.subexpression([left, right]) + else: + # We're creating a projection. + self._advance() + right = self._parse_projection_rhs( + self.BINDING_POWER['dot']) + return ast.value_projection(left, right) + + def _token_led_pipe(self, left): + right = self._expression(self.BINDING_POWER['pipe']) + return ast.pipe(left, right) + + def _token_led_or(self, left): + right = self._expression(self.BINDING_POWER['or']) + return ast.or_expression(left, right) + + def _token_led_and(self, left): + right = self._expression(self.BINDING_POWER['and']) + return ast.and_expression(left, right) + + def _token_led_lparen(self, left): + if left['type'] != 'field': + # 0 - first func arg or closing paren. + # -1 - '(' token + # -2 - invalid function "name". + prev_t = self._lookahead_token(-2) + raise exceptions.ParseError( + prev_t['start'], prev_t['value'], prev_t['type'], + "Invalid function name '%s'" % prev_t['value']) + name = left['value'] + args = [] + while not self._current_token() == 'rparen': + expression = self._expression() + if self._current_token() == 'comma': + self._match('comma') + args.append(expression) + self._match('rparen') + function_node = ast.function_expression(name, args) + return function_node + + def _token_led_filter(self, left): + # Filters are projections. + condition = self._expression(0) + self._match('rbracket') + if self._current_token() == 'flatten': + right = ast.identity() + else: + right = self._parse_projection_rhs(self.BINDING_POWER['filter']) + return ast.filter_projection(left, right, condition) + + def _token_led_eq(self, left): + return self._parse_comparator(left, 'eq') + + def _token_led_ne(self, left): + return self._parse_comparator(left, 'ne') + + def _token_led_gt(self, left): + return self._parse_comparator(left, 'gt') + + def _token_led_gte(self, left): + return self._parse_comparator(left, 'gte') + + def _token_led_lt(self, left): + return self._parse_comparator(left, 'lt') + + def _token_led_lte(self, left): + return self._parse_comparator(left, 'lte') + + def _token_led_flatten(self, left): + left = ast.flatten(left) + right = self._parse_projection_rhs( + self.BINDING_POWER['flatten']) + return ast.projection(left, right) + + def _token_led_lbracket(self, left): + token = self._lookahead_token(0) + if token['type'] in ['number', 'colon']: + right = self._parse_index_expression() + if left['type'] == 'index_expression': + # Optimization: if the left node is an index expr, + # we can avoid creating another node and instead just add + # the right node as a child of the left. + left['children'].append(right) + return left + else: + return self._project_if_slice(left, right) + else: + # We have a projection + self._match('star') + self._match('rbracket') + right = self._parse_projection_rhs(self.BINDING_POWER['star']) + return ast.projection(left, right) + + def _project_if_slice(self, left, right): + index_expr = ast.index_expression([left, right]) + if right['type'] == 'slice': + return ast.projection( + index_expr, + self._parse_projection_rhs(self.BINDING_POWER['star'])) + else: + return index_expr + + def _parse_comparator(self, left, comparator): + right = self._expression(self.BINDING_POWER[comparator]) + return ast.comparator(comparator, left, right) + + def _parse_multi_select_list(self): + expressions = [] + while True: + expression = self._expression() + expressions.append(expression) + if self._current_token() == 'rbracket': + break + else: + self._match('comma') + self._match('rbracket') + return ast.multi_select_list(expressions) + + def _parse_multi_select_hash(self): + pairs = [] + while True: + key_token = self._lookahead_token(0) + # Before getting the token value, verify it's + # an identifier. + self._match_multiple_tokens( + token_types=['quoted_identifier', 'unquoted_identifier']) + key_name = key_token['value'] + self._match('colon') + value = self._expression(0) + node = ast.key_val_pair(key_name=key_name, node=value) + pairs.append(node) + if self._current_token() == 'comma': + self._match('comma') + elif self._current_token() == 'rbrace': + self._match('rbrace') + break + return ast.multi_select_dict(nodes=pairs) + + def _parse_projection_rhs(self, binding_power): + # Parse the right hand side of the projection. + if self.BINDING_POWER[self._current_token()] < self._PROJECTION_STOP: + # BP of 10 are all the tokens that stop a projection. + right = ast.identity() + elif self._current_token() == 'lbracket': + right = self._expression(binding_power) + elif self._current_token() == 'filter': + right = self._expression(binding_power) + elif self._current_token() == 'dot': + self._match('dot') + right = self._parse_dot_rhs(binding_power) + else: + self._raise_parse_error_for_token(self._lookahead_token(0), + 'syntax error') + return right + + def _parse_dot_rhs(self, binding_power): + # From the grammar: + # expression '.' ( identifier / + # multi-select-list / + # multi-select-hash / + # function-expression / + # * + # In terms of tokens that means that after a '.', + # you can have: + lookahead = self._current_token() + # Common case "foo.bar", so first check for an identifier. + if lookahead in ['quoted_identifier', 'unquoted_identifier', 'star']: + return self._expression(binding_power) + elif lookahead == 'lbracket': + self._match('lbracket') + return self._parse_multi_select_list() + elif lookahead == 'lbrace': + self._match('lbrace') + return self._parse_multi_select_hash() + else: + t = self._lookahead_token(0) + allowed = ['quoted_identifier', 'unquoted_identifier', + 'lbracket', 'lbrace'] + msg = ( + "Expecting: %s, got: %s" % (allowed, t['type']) + ) + self._raise_parse_error_for_token(t, msg) + + def _error_nud_token(self, token): + if token['type'] == 'eof': + raise exceptions.IncompleteExpressionError( + token['start'], token['value'], token['type']) + self._raise_parse_error_for_token(token, 'invalid token') + + def _error_led_token(self, token): + self._raise_parse_error_for_token(token, 'invalid token') + + def _match(self, token_type=None): + # inline'd self._current_token() + if self._current_token() == token_type: + # inline'd self._advance() + self._advance() + else: + self._raise_parse_error_maybe_eof( + token_type, self._lookahead_token(0)) + + def _match_multiple_tokens(self, token_types): + if self._current_token() not in token_types: + self._raise_parse_error_maybe_eof( + token_types, self._lookahead_token(0)) + self._advance() + + def _advance(self): + self._index += 1 + + def _current_token(self): + return self._tokens[self._index]['type'] + + def _lookahead(self, number): + return self._tokens[self._index + number]['type'] + + def _lookahead_token(self, number): + return self._tokens[self._index + number] + + def _raise_parse_error_for_token(self, token, reason): + lex_position = token['start'] + actual_value = token['value'] + actual_type = token['type'] + raise exceptions.ParseError(lex_position, actual_value, + actual_type, reason) + + def _raise_parse_error_maybe_eof(self, expected_type, token): + lex_position = token['start'] + actual_value = token['value'] + actual_type = token['type'] + if actual_type == 'eof': + raise exceptions.IncompleteExpressionError( + lex_position, actual_value, actual_type) + message = 'Expecting: %s, got: %s' % (expected_type, + actual_type) + raise exceptions.ParseError( + lex_position, actual_value, actual_type, message) + + def _free_cache_entries(self): + for key in random.sample(self._CACHE.keys(), int(self._MAX_SIZE / 2)): + self._CACHE.pop(key, None) + + @classmethod + def purge(cls): + """Clear the expression compilation cache.""" + cls._CACHE.clear() + + +@with_repr_method +class ParsedResult(object): + def __init__(self, expression, parsed): + self.expression = expression + self.parsed = parsed + + def search(self, value, options=None): + interpreter = visitor.TreeInterpreter(options) + result = interpreter.visit(self.parsed, value) + return result + + def _render_dot_file(self): + """Render the parsed AST as a dot file. + + Note that this is marked as an internal method because + the AST is an implementation detail and is subject + to change. This method can be used to help troubleshoot + or for development purposes, but is not considered part + of the public supported API. Use at your own risk. + + """ + renderer = visitor.GraphvizVisitor() + contents = renderer.visit(self.parsed) + return contents + + def __repr__(self): + return repr(self.parsed) diff --git a/WebCrawler/venv/Lib/site-packages/jmespath/visitor.py b/WebCrawler/venv/Lib/site-packages/jmespath/visitor.py new file mode 100644 index 0000000..b3e846b --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/jmespath/visitor.py @@ -0,0 +1,328 @@ +import operator + +from jmespath import functions +from jmespath.compat import string_type +from numbers import Number + + +def _equals(x, y): + if _is_special_integer_case(x, y): + return False + else: + return x == y + + +def _is_special_integer_case(x, y): + # We need to special case comparing 0 or 1 to + # True/False. While normally comparing any + # integer other than 0/1 to True/False will always + # return False. However 0/1 have this: + # >>> 0 == True + # False + # >>> 0 == False + # True + # >>> 1 == True + # True + # >>> 1 == False + # False + # + # Also need to consider that: + # >>> 0 in [True, False] + # True + if type(x) is int and (x == 0 or x == 1): + return y is True or y is False + elif type(y) is int and (y == 0 or y == 1): + return x is True or x is False + + +def _is_comparable(x): + # The spec doesn't officially support string types yet, + # but enough people are relying on this behavior that + # it's been added back. This should eventually become + # part of the official spec. + return _is_actual_number(x) or isinstance(x, string_type) + + +def _is_actual_number(x): + # We need to handle python's quirkiness with booleans, + # specifically: + # + # >>> isinstance(False, int) + # True + # >>> isinstance(True, int) + # True + if x is True or x is False: + return False + return isinstance(x, Number) + + +class Options(object): + """Options to control how a JMESPath function is evaluated.""" + def __init__(self, dict_cls=None, custom_functions=None): + #: The class to use when creating a dict. The interpreter + # may create dictionaries during the evaluation of a JMESPath + # expression. For example, a multi-select hash will + # create a dictionary. By default we use a dict() type. + # You can set this value to change what dict type is used. + # The most common reason you would change this is if you + # want to set a collections.OrderedDict so that you can + # have predictable key ordering. + self.dict_cls = dict_cls + self.custom_functions = custom_functions + + +class _Expression(object): + def __init__(self, expression, interpreter): + self.expression = expression + self.interpreter = interpreter + + def visit(self, node, *args, **kwargs): + return self.interpreter.visit(node, *args, **kwargs) + + +class Visitor(object): + def __init__(self): + self._method_cache = {} + + def visit(self, node, *args, **kwargs): + node_type = node['type'] + method = self._method_cache.get(node_type) + if method is None: + method = getattr( + self, 'visit_%s' % node['type'], self.default_visit) + self._method_cache[node_type] = method + return method(node, *args, **kwargs) + + def default_visit(self, node, *args, **kwargs): + raise NotImplementedError("default_visit") + + +class TreeInterpreter(Visitor): + COMPARATOR_FUNC = { + 'eq': _equals, + 'ne': lambda x, y: not _equals(x, y), + 'lt': operator.lt, + 'gt': operator.gt, + 'lte': operator.le, + 'gte': operator.ge + } + _EQUALITY_OPS = ['eq', 'ne'] + MAP_TYPE = dict + + def __init__(self, options=None): + super(TreeInterpreter, self).__init__() + self._dict_cls = self.MAP_TYPE + if options is None: + options = Options() + self._options = options + if options.dict_cls is not None: + self._dict_cls = self._options.dict_cls + if options.custom_functions is not None: + self._functions = self._options.custom_functions + else: + self._functions = functions.Functions() + + def default_visit(self, node, *args, **kwargs): + raise NotImplementedError(node['type']) + + def visit_subexpression(self, node, value): + result = value + for node in node['children']: + result = self.visit(node, result) + return result + + def visit_field(self, node, value): + try: + return value.get(node['value']) + except AttributeError: + return None + + def visit_comparator(self, node, value): + # Common case: comparator is == or != + comparator_func = self.COMPARATOR_FUNC[node['value']] + if node['value'] in self._EQUALITY_OPS: + return comparator_func( + self.visit(node['children'][0], value), + self.visit(node['children'][1], value) + ) + else: + # Ordering operators are only valid for numbers. + # Evaluating any other type with a comparison operator + # will yield a None value. + left = self.visit(node['children'][0], value) + right = self.visit(node['children'][1], value) + num_types = (int, float) + if not (_is_comparable(left) and + _is_comparable(right)): + return None + return comparator_func(left, right) + + def visit_current(self, node, value): + return value + + def visit_expref(self, node, value): + return _Expression(node['children'][0], self) + + def visit_function_expression(self, node, value): + resolved_args = [] + for child in node['children']: + current = self.visit(child, value) + resolved_args.append(current) + return self._functions.call_function(node['value'], resolved_args) + + def visit_filter_projection(self, node, value): + base = self.visit(node['children'][0], value) + if not isinstance(base, list): + return None + comparator_node = node['children'][2] + collected = [] + for element in base: + if self._is_true(self.visit(comparator_node, element)): + current = self.visit(node['children'][1], element) + if current is not None: + collected.append(current) + return collected + + def visit_flatten(self, node, value): + base = self.visit(node['children'][0], value) + if not isinstance(base, list): + # Can't flatten the object if it's not a list. + return None + merged_list = [] + for element in base: + if isinstance(element, list): + merged_list.extend(element) + else: + merged_list.append(element) + return merged_list + + def visit_identity(self, node, value): + return value + + def visit_index(self, node, value): + # Even though we can index strings, we don't + # want to support that. + if not isinstance(value, list): + return None + try: + return value[node['value']] + except IndexError: + return None + + def visit_index_expression(self, node, value): + result = value + for node in node['children']: + result = self.visit(node, result) + return result + + def visit_slice(self, node, value): + if not isinstance(value, list): + return None + s = slice(*node['children']) + return value[s] + + def visit_key_val_pair(self, node, value): + return self.visit(node['children'][0], value) + + def visit_literal(self, node, value): + return node['value'] + + def visit_multi_select_dict(self, node, value): + if value is None: + return None + collected = self._dict_cls() + for child in node['children']: + collected[child['value']] = self.visit(child, value) + return collected + + def visit_multi_select_list(self, node, value): + if value is None: + return None + collected = [] + for child in node['children']: + collected.append(self.visit(child, value)) + return collected + + def visit_or_expression(self, node, value): + matched = self.visit(node['children'][0], value) + if self._is_false(matched): + matched = self.visit(node['children'][1], value) + return matched + + def visit_and_expression(self, node, value): + matched = self.visit(node['children'][0], value) + if self._is_false(matched): + return matched + return self.visit(node['children'][1], value) + + def visit_not_expression(self, node, value): + original_result = self.visit(node['children'][0], value) + if type(original_result) is int and original_result == 0: + # Special case for 0, !0 should be false, not true. + # 0 is not a special cased integer in jmespath. + return False + return not original_result + + def visit_pipe(self, node, value): + result = value + for node in node['children']: + result = self.visit(node, result) + return result + + def visit_projection(self, node, value): + base = self.visit(node['children'][0], value) + if not isinstance(base, list): + return None + collected = [] + for element in base: + current = self.visit(node['children'][1], element) + if current is not None: + collected.append(current) + return collected + + def visit_value_projection(self, node, value): + base = self.visit(node['children'][0], value) + try: + base = base.values() + except AttributeError: + return None + collected = [] + for element in base: + current = self.visit(node['children'][1], element) + if current is not None: + collected.append(current) + return collected + + def _is_false(self, value): + # This looks weird, but we're explicitly using equality checks + # because the truth/false values are different between + # python and jmespath. + return (value == '' or value == [] or value == {} or value is None or + value is False) + + def _is_true(self, value): + return not self._is_false(value) + + +class GraphvizVisitor(Visitor): + def __init__(self): + super(GraphvizVisitor, self).__init__() + self._lines = [] + self._count = 1 + + def visit(self, node, *args, **kwargs): + self._lines.append('digraph AST {') + current = '%s%s' % (node['type'], self._count) + self._count += 1 + self._visit(node, current) + self._lines.append('}') + return '\n'.join(self._lines) + + def _visit(self, node, current): + self._lines.append('%s [label="%s(%s)"]' % ( + current, node['type'], node.get('value', ''))) + for child in node.get('children', []): + child_name = '%s%s' % (child['type'], self._count) + self._count += 1 + self._lines.append(' %s -> %s' % (current, child_name)) + self._visit(child, child_name) diff --git a/WebCrawler/venv/Lib/site-packages/lazy_object_proxy-1.4.3.dist-info/AUTHORS.rst b/WebCrawler/venv/Lib/site-packages/lazy_object_proxy-1.4.3.dist-info/AUTHORS.rst new file mode 100644 index 0000000..dbc0324 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/lazy_object_proxy-1.4.3.dist-info/AUTHORS.rst @@ -0,0 +1,10 @@ + +Authors +======= + +* Ionel Cristian Mărieș - https://blog.ionelmc.ro +* Alvin Chow - https://github.com/alvinchow86 +* Astrum Kuo - https://github.com/xowenx +* Erik M. Bray - http://iguananaut.net +* Ran Benita - https://github.com/bluetech +* "hugovk" - https://github.com/hugovk diff --git a/WebCrawler/venv/Lib/site-packages/lazy_object_proxy-1.4.3.dist-info/INSTALLER b/WebCrawler/venv/Lib/site-packages/lazy_object_proxy-1.4.3.dist-info/INSTALLER new file mode 100644 index 0000000..a1b589e --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/lazy_object_proxy-1.4.3.dist-info/INSTALLER @@ -0,0 +1 @@ +pip diff --git a/WebCrawler/venv/Lib/site-packages/lazy_object_proxy-1.4.3.dist-info/LICENSE b/WebCrawler/venv/Lib/site-packages/lazy_object_proxy-1.4.3.dist-info/LICENSE new file mode 100644 index 0000000..de39b84 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/lazy_object_proxy-1.4.3.dist-info/LICENSE @@ -0,0 +1,21 @@ +BSD 2-Clause License + +Copyright (c) 2014-2019, Ionel Cristian Mărieș +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the +following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following +disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following +disclaimer in the documentation and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, +INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, +WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/WebCrawler/venv/Lib/site-packages/lazy_object_proxy-1.4.3.dist-info/METADATA b/WebCrawler/venv/Lib/site-packages/lazy_object_proxy-1.4.3.dist-info/METADATA new file mode 100644 index 0000000..6b4b830 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/lazy_object_proxy-1.4.3.dist-info/METADATA @@ -0,0 +1,166 @@ +Metadata-Version: 2.1 +Name: lazy-object-proxy +Version: 1.4.3 +Summary: A fast and thorough lazy object proxy. +Home-page: https://github.com/ionelmc/python-lazy-object-proxy +Author: Ionel Cristian Mărieș +Author-email: contact@ionelmc.ro +License: BSD-2-Clause +Project-URL: Documentation, https://python-lazy-object-proxy.readthedocs.io/ +Project-URL: Changelog, https://python-lazy-object-proxy.readthedocs.io/en/latest/changelog.html +Project-URL: Issue Tracker, https://github.com/ionelmc/python-lazy-object-proxy/issues +Platform: UNKNOWN +Classifier: Development Status :: 5 - Production/Stable +Classifier: Intended Audience :: Developers +Classifier: License :: OSI Approved :: BSD License +Classifier: Operating System :: Unix +Classifier: Operating System :: POSIX +Classifier: Operating System :: Microsoft :: Windows +Classifier: Programming Language :: Python +Classifier: Programming Language :: Python :: 2.7 +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3.4 +Classifier: Programming Language :: Python :: 3.5 +Classifier: Programming Language :: Python :: 3.6 +Classifier: Programming Language :: Python :: 3.7 +Classifier: Programming Language :: Python :: 3.8 +Classifier: Programming Language :: Python :: Implementation :: CPython +Classifier: Programming Language :: Python :: Implementation :: PyPy +Classifier: Topic :: Utilities +Requires-Python: >=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.* + +======== +Overview +======== + + + +A fast and thorough lazy object proxy. + +* Free software: BSD 2-Clause License + +Note that this is based on `wrapt`_'s ObjectProxy with one big change: it calls a function the first time the proxy object is +used, while `wrapt.ObjectProxy` just forwards the method calls to the target object. + +In other words, you use `lazy-object-proxy` when you only have the object way later and you use `wrapt.ObjectProxy` when you +want to override few methods (by subclassing) and forward everything else to the target object. + +Example:: + + import lazy_object_proxy + + def expensive_func(): + from time import sleep + print('starting calculation') + # just as example for a very slow computation + sleep(2) + print('finished calculation') + # return the result of the calculation + return 10 + + obj = lazy_object_proxy.Proxy(expensive_func) + # function is called only when object is actually used + print(obj) # now expensive_func is called + + print(obj) # the result without calling the expensive_func + +Installation +============ + +:: + + pip install lazy-object-proxy + +Documentation +============= + +https://python-lazy-object-proxy.readthedocs.io/ + +Development +=========== + +To run the all tests run:: + + tox + +Acknowledgements +================ + +This project is based on some code from `wrapt`_ as you can see in the git history. + +.. _wrapt: https://github.com/GrahamDumpleton/wrapt + + +Changelog +========= + +1.4.3 (2019-10-26) +------------------ + +* Added binary wheels for Python 3.8. +* Fixed license metadata. + +1.4.2 (2019-08-22) +------------------ + +* Included a ``pyproject.toml`` to allow users install the sdist with old python/setuptools, as the + setuptools-scm dep will be fetched by pip instead of setuptools. + Fixes `#30 `_. + +1.4.1 (2019-05-10) +------------------ + +* Fixed wheels being built with ``-coverage`` cflags. No more issues about bogus ``cext.gcda`` files. +* Removed useless C file from wheels. +* Changed ``setup.py`` to use setuptools-scm. + +1.4.0 (2019-05-05) +------------------ + +* Fixed ``__mod__`` for the slots backend. Contributed by Ran Benita in + `#28 `_. +* Dropped support for Python 2.6 and 3.3. Contributed by "hugovk" in + `#24 `_. + +1.3.1 (2017-05-05) +------------------ + +* Fix broken release (``sdist`` had a broken ``MANIFEST.in``). + +1.3.0 (2017-05-02) +------------------ + +* Speed up arithmetic operations involving ``cext.Proxy`` subclasses. + +1.2.2 (2016-04-14) +------------------ + +* Added `manylinux `_ wheels. +* Minor cleanup in readme. + +1.2.1 (2015-08-18) +------------------ + +* Fix a memory leak (the wrapped object would get bogus references). Contributed by Astrum Kuo in + `#10 `_. + +1.2.0 (2015-07-06) +------------------ + +* Don't instantiate the object when __repr__ is called. This aids with debugging (allows one to see exactly in + what state the proxy is). + +1.1.0 (2015-07-05) +------------------ + +* Added support for pickling. The pickled value is going to be the wrapped object *without* any Proxy container. +* Fixed a memory management issue in the C extension (reference cycles weren't garbage collected due to improper + handling in the C extension). Contributed by Alvin Chow in + `#8 `_. + +1.0.2 (2015-04-11) +----------------------------------------- + +* First release on PyPI. + + diff --git a/WebCrawler/venv/Lib/site-packages/lazy_object_proxy-1.4.3.dist-info/RECORD b/WebCrawler/venv/Lib/site-packages/lazy_object_proxy-1.4.3.dist-info/RECORD new file mode 100644 index 0000000..a3c58d3 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/lazy_object_proxy-1.4.3.dist-info/RECORD @@ -0,0 +1,19 @@ +lazy_object_proxy-1.4.3.dist-info/AUTHORS.rst,sha256=8CeCjODba0S8UczLyZBPhpO_J6NMZ9Hz_fE1A1uNe9Y,278 +lazy_object_proxy-1.4.3.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4 +lazy_object_proxy-1.4.3.dist-info/LICENSE,sha256=W-1KNkH2bsSNuN7SNqKV8z2H0CkxXzYXZVhUzw1wxUA,1329 +lazy_object_proxy-1.4.3.dist-info/METADATA,sha256=Y2X63wcFbQT4yI3zKpRFwfpA_TCpB6U79MPmRGCMJT0,5088 +lazy_object_proxy-1.4.3.dist-info/RECORD,, +lazy_object_proxy-1.4.3.dist-info/WHEEL,sha256=jr7ubY0Lkz_yXH9FfFe9PTtLhGOsf62dZkNvTYrJINE,100 +lazy_object_proxy-1.4.3.dist-info/top_level.txt,sha256=UNH-FQB-j_8bYqPz3gD90kHvaC42TQqY0thHSnbaa0k,18 +lazy_object_proxy/__init__.py,sha256=pMqxzToF24DuzOltm-Q8nZ3jNDXOaSQcJmiNArdUrlU,410 +lazy_object_proxy/__pycache__/__init__.cpython-39.pyc,, +lazy_object_proxy/__pycache__/_version.cpython-39.pyc,, +lazy_object_proxy/__pycache__/compat.cpython-39.pyc,, +lazy_object_proxy/__pycache__/simple.cpython-39.pyc,, +lazy_object_proxy/__pycache__/slots.cpython-39.pyc,, +lazy_object_proxy/__pycache__/utils.cpython-39.pyc,, +lazy_object_proxy/_version.py,sha256=BrsSYfSHcNZPgRwz64o6JwYujFV62qIxBKDB1FXVlb4,147 +lazy_object_proxy/compat.py,sha256=DY3HbKwbrbeKY6tkXRNRLJ1go6HJb8HUwWqyw3T5g_g,196 +lazy_object_proxy/simple.py,sha256=guacy8_QbJeBs7vXpPPVoDVkDNXOZ86xyS1dtAeKvOs,8216 +lazy_object_proxy/slots.py,sha256=9DilWUINScpZN26NwmRtscTtaqaEwtCfIoriLq5Nz24,11359 +lazy_object_proxy/utils.py,sha256=x4XTrtlp_mDTWO_EOq_ILIOv2Qol8RLMnRm5M8l3OfU,291 diff --git a/WebCrawler/venv/Lib/site-packages/lazy_object_proxy-1.4.3.dist-info/WHEEL b/WebCrawler/venv/Lib/site-packages/lazy_object_proxy-1.4.3.dist-info/WHEEL new file mode 100644 index 0000000..d1267fc --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/lazy_object_proxy-1.4.3.dist-info/WHEEL @@ -0,0 +1,5 @@ +Wheel-Version: 1.0 +Generator: bdist_wheel (0.36.2) +Root-Is-Purelib: false +Tag: cp39-cp39-win_amd64 + diff --git a/WebCrawler/venv/Lib/site-packages/lazy_object_proxy-1.4.3.dist-info/top_level.txt b/WebCrawler/venv/Lib/site-packages/lazy_object_proxy-1.4.3.dist-info/top_level.txt new file mode 100644 index 0000000..bdf032e --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/lazy_object_proxy-1.4.3.dist-info/top_level.txt @@ -0,0 +1 @@ +lazy_object_proxy diff --git a/WebCrawler/venv/Lib/site-packages/lazy_object_proxy/__init__.py b/WebCrawler/venv/Lib/site-packages/lazy_object_proxy/__init__.py new file mode 100644 index 0000000..e9a9a76 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/lazy_object_proxy/__init__.py @@ -0,0 +1,23 @@ +try: + import copy_reg as copyreg +except ImportError: + import copyreg + +from .utils import identity + +copyreg.constructor(identity) + +try: + from .cext import Proxy + from .cext import identity +except ImportError: + from .slots import Proxy +else: + copyreg.constructor(identity) + +try: + from ._version import version as __version__ +except ImportError: + __version__ = '1.4.3' + +__all__ = "Proxy", diff --git a/WebCrawler/venv/Lib/site-packages/lazy_object_proxy/__pycache__/__init__.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/lazy_object_proxy/__pycache__/__init__.cpython-39.pyc new file mode 100644 index 0000000..9ac2af4 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/lazy_object_proxy/__pycache__/__init__.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/lazy_object_proxy/__pycache__/_version.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/lazy_object_proxy/__pycache__/_version.cpython-39.pyc new file mode 100644 index 0000000..6f389c0 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/lazy_object_proxy/__pycache__/_version.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/lazy_object_proxy/__pycache__/compat.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/lazy_object_proxy/__pycache__/compat.cpython-39.pyc new file mode 100644 index 0000000..69099dd Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/lazy_object_proxy/__pycache__/compat.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/lazy_object_proxy/__pycache__/simple.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/lazy_object_proxy/__pycache__/simple.cpython-39.pyc new file mode 100644 index 0000000..e58ca4c Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/lazy_object_proxy/__pycache__/simple.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/lazy_object_proxy/__pycache__/slots.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/lazy_object_proxy/__pycache__/slots.cpython-39.pyc new file mode 100644 index 0000000..acf023b Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/lazy_object_proxy/__pycache__/slots.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/lazy_object_proxy/__pycache__/utils.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/lazy_object_proxy/__pycache__/utils.cpython-39.pyc new file mode 100644 index 0000000..85662da Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/lazy_object_proxy/__pycache__/utils.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/lazy_object_proxy/_version.py b/WebCrawler/venv/Lib/site-packages/lazy_object_proxy/_version.py new file mode 100644 index 0000000..2c3d3dc --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/lazy_object_proxy/_version.py @@ -0,0 +1,5 @@ +# coding: utf-8 +# file generated by setuptools_scm +# don't change, don't track in version control +version = '1.4.3' +version_tuple = (1, 4, 3) diff --git a/WebCrawler/venv/Lib/site-packages/lazy_object_proxy/compat.py b/WebCrawler/venv/Lib/site-packages/lazy_object_proxy/compat.py new file mode 100644 index 0000000..dc6edfa --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/lazy_object_proxy/compat.py @@ -0,0 +1,9 @@ +import sys + +PY2 = sys.version_info[0] == 2 +PY3 = sys.version_info[0] == 3 + + +def with_metaclass(meta, *bases): + """Create a base class with a metaclass.""" + return meta("NewBase", bases, {}) diff --git a/WebCrawler/venv/Lib/site-packages/lazy_object_proxy/simple.py b/WebCrawler/venv/Lib/site-packages/lazy_object_proxy/simple.py new file mode 100644 index 0000000..24b1339 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/lazy_object_proxy/simple.py @@ -0,0 +1,246 @@ +import operator + +from .compat import PY2 +from .compat import PY3 +from .compat import with_metaclass +from .utils import cached_property +from .utils import identity + + +def make_proxy_method(code): + def proxy_wrapper(self, *args): + return code(self.__wrapped__, *args) + + return proxy_wrapper + + +class _ProxyMethods(object): + # We use properties to override the values of __module__ and + # __doc__. If we add these in ObjectProxy, the derived class + # __dict__ will still be setup to have string variants of these + # attributes and the rules of descriptors means that they appear to + # take precedence over the properties in the base class. To avoid + # that, we copy the properties into the derived class type itself + # via a meta class. In that way the properties will always take + # precedence. + + @property + def __module__(self): + return self.__wrapped__.__module__ + + @__module__.setter + def __module__(self, value): + self.__wrapped__.__module__ = value + + @property + def __doc__(self): + return self.__wrapped__.__doc__ + + @__doc__.setter + def __doc__(self, value): + self.__wrapped__.__doc__ = value + + # Need to also propagate the special __weakref__ attribute for case + # where decorating classes which will define this. If do not define + # it and use a function like inspect.getmembers() on a decorator + # class it will fail. This can't be in the derived classes. + + @property + def __weakref__(self): + return self.__wrapped__.__weakref__ + + +class _ProxyMetaType(type): + def __new__(cls, name, bases, dictionary): + # Copy our special properties into the class so that they + # always take precedence over attributes of the same name added + # during construction of a derived class. This is to save + # duplicating the implementation for them in all derived classes. + + dictionary.update(vars(_ProxyMethods)) + dictionary.pop('__dict__') + + return type.__new__(cls, name, bases, dictionary) + + +class Proxy(with_metaclass(_ProxyMetaType)): + __factory__ = None + + def __init__(self, factory): + self.__dict__['__factory__'] = factory + + @cached_property + def __wrapped__(self): + self = self.__dict__ + if '__factory__' in self: + factory = self['__factory__'] + return factory() + else: + raise ValueError("Proxy hasn't been initiated: __factory__ is missing.") + + __name__ = property(make_proxy_method(operator.attrgetter('__name__'))) + __class__ = property(make_proxy_method(operator.attrgetter('__class__'))) + __annotations__ = property(make_proxy_method(operator.attrgetter('__anotations__'))) + __dir__ = make_proxy_method(dir) + __str__ = make_proxy_method(str) + + if PY3: + __bytes__ = make_proxy_method(bytes) + + def __repr__(self, __getattr__=object.__getattribute__): + if '__wrapped__' in self.__dict__: + return '<{} at 0x{:x} wrapping {!r} at 0x{:x} with factory {!r}>'.format( + type(self).__name__, id(self), + self.__wrapped__, id(self.__wrapped__), + self.__factory__ + ) + else: + return '<{} at 0x{:x} with factory {!r}>'.format( + type(self).__name__, id(self), + self.__factory__ + ) + + __reversed__ = make_proxy_method(reversed) + + if PY3: + __round__ = make_proxy_method(round) + + __lt__ = make_proxy_method(operator.lt) + __le__ = make_proxy_method(operator.le) + __eq__ = make_proxy_method(operator.eq) + __ne__ = make_proxy_method(operator.ne) + __gt__ = make_proxy_method(operator.gt) + __ge__ = make_proxy_method(operator.ge) + __hash__ = make_proxy_method(hash) + __nonzero__ = make_proxy_method(bool) + __bool__ = make_proxy_method(bool) + + def __setattr__(self, name, value): + if hasattr(type(self), name): + self.__dict__[name] = value + else: + setattr(self.__wrapped__, name, value) + + def __getattr__(self, name): + if name in ('__wrapped__', '__factory__'): + raise AttributeError(name) + else: + return getattr(self.__wrapped__, name) + + def __delattr__(self, name): + if hasattr(type(self), name): + del self.__dict__[name] + else: + delattr(self.__wrapped__, name) + + __add__ = make_proxy_method(operator.add) + __sub__ = make_proxy_method(operator.sub) + __mul__ = make_proxy_method(operator.mul) + __div__ = make_proxy_method(operator.div if PY2 else operator.truediv) + __truediv__ = make_proxy_method(operator.truediv) + __floordiv__ = make_proxy_method(operator.floordiv) + __mod__ = make_proxy_method(operator.mod) + __divmod__ = make_proxy_method(divmod) + __pow__ = make_proxy_method(pow) + __lshift__ = make_proxy_method(operator.lshift) + __rshift__ = make_proxy_method(operator.rshift) + __and__ = make_proxy_method(operator.and_) + __xor__ = make_proxy_method(operator.xor) + __or__ = make_proxy_method(operator.or_) + + def __radd__(self, other): + return other + self.__wrapped__ + + def __rsub__(self, other): + return other - self.__wrapped__ + + def __rmul__(self, other): + return other * self.__wrapped__ + + def __rdiv__(self, other): + return operator.div(other, self.__wrapped__) + + def __rtruediv__(self, other): + return operator.truediv(other, self.__wrapped__) + + def __rfloordiv__(self, other): + return other // self.__wrapped__ + + def __rmod__(self, other): + return other % self.__wrapped__ + + def __rdivmod__(self, other): + return divmod(other, self.__wrapped__) + + def __rpow__(self, other, *args): + return pow(other, self.__wrapped__, *args) + + def __rlshift__(self, other): + return other << self.__wrapped__ + + def __rrshift__(self, other): + return other >> self.__wrapped__ + + def __rand__(self, other): + return other & self.__wrapped__ + + def __rxor__(self, other): + return other ^ self.__wrapped__ + + def __ror__(self, other): + return other | self.__wrapped__ + + __iadd__ = make_proxy_method(operator.iadd) + __isub__ = make_proxy_method(operator.isub) + __imul__ = make_proxy_method(operator.imul) + __idiv__ = make_proxy_method(operator.idiv if PY2 else operator.itruediv) + __itruediv__ = make_proxy_method(operator.itruediv) + __ifloordiv__ = make_proxy_method(operator.ifloordiv) + __imod__ = make_proxy_method(operator.imod) + __ipow__ = make_proxy_method(operator.ipow) + __ilshift__ = make_proxy_method(operator.ilshift) + __irshift__ = make_proxy_method(operator.irshift) + __iand__ = make_proxy_method(operator.iand) + __ixor__ = make_proxy_method(operator.ixor) + __ior__ = make_proxy_method(operator.ior) + __neg__ = make_proxy_method(operator.neg) + __pos__ = make_proxy_method(operator.pos) + __abs__ = make_proxy_method(operator.abs) + __invert__ = make_proxy_method(operator.invert) + + __int__ = make_proxy_method(int) + + if PY2: + __long__ = make_proxy_method(long) # noqa + + __float__ = make_proxy_method(float) + __oct__ = make_proxy_method(oct) + __hex__ = make_proxy_method(hex) + __index__ = make_proxy_method(operator.index) + __len__ = make_proxy_method(len) + __contains__ = make_proxy_method(operator.contains) + __getitem__ = make_proxy_method(operator.getitem) + __setitem__ = make_proxy_method(operator.setitem) + __delitem__ = make_proxy_method(operator.delitem) + + if PY2: + __getslice__ = make_proxy_method(operator.getslice) + __setslice__ = make_proxy_method(operator.setslice) + __delslice__ = make_proxy_method(operator.delslice) + + def __enter__(self): + return self.__wrapped__.__enter__() + + def __exit__(self, *args, **kwargs): + return self.__wrapped__.__exit__(*args, **kwargs) + + __iter__ = make_proxy_method(iter) + + def __call__(self, *args, **kwargs): + return self.__wrapped__(*args, **kwargs) + + def __reduce__(self): + return identity, (self.__wrapped__,) + + def __reduce_ex__(self, protocol): + return identity, (self.__wrapped__,) diff --git a/WebCrawler/venv/Lib/site-packages/lazy_object_proxy/slots.py b/WebCrawler/venv/Lib/site-packages/lazy_object_proxy/slots.py new file mode 100644 index 0000000..efb08db --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/lazy_object_proxy/slots.py @@ -0,0 +1,414 @@ +import operator + +from .compat import PY2 +from .compat import PY3 +from .compat import with_metaclass +from .utils import identity + + +class _ProxyMethods(object): + # We use properties to override the values of __module__ and + # __doc__. If we add these in ObjectProxy, the derived class + # __dict__ will still be setup to have string variants of these + # attributes and the rules of descriptors means that they appear to + # take precedence over the properties in the base class. To avoid + # that, we copy the properties into the derived class type itself + # via a meta class. In that way the properties will always take + # precedence. + + @property + def __module__(self): + return self.__wrapped__.__module__ + + @__module__.setter + def __module__(self, value): + self.__wrapped__.__module__ = value + + @property + def __doc__(self): + return self.__wrapped__.__doc__ + + @__doc__.setter + def __doc__(self, value): + self.__wrapped__.__doc__ = value + + # We similar use a property for __dict__. We need __dict__ to be + # explicit to ensure that vars() works as expected. + + @property + def __dict__(self): + return self.__wrapped__.__dict__ + + # Need to also propagate the special __weakref__ attribute for case + # where decorating classes which will define this. If do not define + # it and use a function like inspect.getmembers() on a decorator + # class it will fail. This can't be in the derived classes. + + @property + def __weakref__(self): + return self.__wrapped__.__weakref__ + + +class _ProxyMetaType(type): + def __new__(cls, name, bases, dictionary): + # Copy our special properties into the class so that they + # always take precedence over attributes of the same name added + # during construction of a derived class. This is to save + # duplicating the implementation for them in all derived classes. + + dictionary.update(vars(_ProxyMethods)) + + return type.__new__(cls, name, bases, dictionary) + + +class Proxy(with_metaclass(_ProxyMetaType)): + """ + A proxy implementation in pure Python, using slots. You can subclass this to add + local methods or attributes, or enable __dict__. + + The most important internals: + + * ``__factory__`` is the callback that "materializes" the object we proxy to. + * ``__target__`` will contain the object we proxy to, once it's "materialized". + * ``__wrapped__`` is a property that does either: + + * return ``__target__`` if it's set. + * calls ``__factory__``, saves result to ``__target__`` and returns said result. + """ + + __slots__ = '__target__', '__factory__' + + def __init__(self, factory): + object.__setattr__(self, '__factory__', factory) + + @property + def __wrapped__(self, __getattr__=object.__getattribute__, __setattr__=object.__setattr__, + __delattr__=object.__delattr__): + try: + return __getattr__(self, '__target__') + except AttributeError: + try: + factory = __getattr__(self, '__factory__') + except AttributeError: + raise ValueError("Proxy hasn't been initiated: __factory__ is missing.") + target = factory() + __setattr__(self, '__target__', target) + return target + + @__wrapped__.deleter + def __wrapped__(self, __delattr__=object.__delattr__): + __delattr__(self, '__target__') + + @__wrapped__.setter + def __wrapped__(self, target, __setattr__=object.__setattr__): + __setattr__(self, '__target__', target) + + @property + def __name__(self): + return self.__wrapped__.__name__ + + @__name__.setter + def __name__(self, value): + self.__wrapped__.__name__ = value + + @property + def __class__(self): + return self.__wrapped__.__class__ + + @__class__.setter # noqa + def __class__(self, value): + self.__wrapped__.__class__ = value + + @property + def __annotations__(self): + return self.__wrapped__.__anotations__ + + @__annotations__.setter + def __annotations__(self, value): + self.__wrapped__.__annotations__ = value + + def __dir__(self): + return dir(self.__wrapped__) + + def __str__(self): + return str(self.__wrapped__) + + if PY3: + def __bytes__(self): + return bytes(self.__wrapped__) + + def __repr__(self, __getattr__=object.__getattribute__): + try: + target = __getattr__(self, '__target__') + except AttributeError: + return '<{} at 0x{:x} with factory {!r}>'.format( + type(self).__name__, id(self), + self.__factory__ + ) + else: + return '<{} at 0x{:x} wrapping {!r} at 0x{:x} with factory {!r}>'.format( + type(self).__name__, id(self), + target, id(target), + self.__factory__ + ) + + def __reversed__(self): + return reversed(self.__wrapped__) + + if PY3: + def __round__(self): + return round(self.__wrapped__) + + def __lt__(self, other): + return self.__wrapped__ < other + + def __le__(self, other): + return self.__wrapped__ <= other + + def __eq__(self, other): + return self.__wrapped__ == other + + def __ne__(self, other): + return self.__wrapped__ != other + + def __gt__(self, other): + return self.__wrapped__ > other + + def __ge__(self, other): + return self.__wrapped__ >= other + + def __hash__(self): + return hash(self.__wrapped__) + + def __nonzero__(self): + return bool(self.__wrapped__) + + def __bool__(self): + return bool(self.__wrapped__) + + def __setattr__(self, name, value, __setattr__=object.__setattr__): + if hasattr(type(self), name): + __setattr__(self, name, value) + else: + setattr(self.__wrapped__, name, value) + + def __getattr__(self, name): + if name in ('__wrapped__', '__factory__'): + raise AttributeError(name) + else: + return getattr(self.__wrapped__, name) + + def __delattr__(self, name, __delattr__=object.__delattr__): + if hasattr(type(self), name): + __delattr__(self, name) + else: + delattr(self.__wrapped__, name) + + def __add__(self, other): + return self.__wrapped__ + other + + def __sub__(self, other): + return self.__wrapped__ - other + + def __mul__(self, other): + return self.__wrapped__ * other + + def __div__(self, other): + return operator.div(self.__wrapped__, other) + + def __truediv__(self, other): + return operator.truediv(self.__wrapped__, other) + + def __floordiv__(self, other): + return self.__wrapped__ // other + + def __mod__(self, other): + return self.__wrapped__ % other + + def __divmod__(self, other): + return divmod(self.__wrapped__, other) + + def __pow__(self, other, *args): + return pow(self.__wrapped__, other, *args) + + def __lshift__(self, other): + return self.__wrapped__ << other + + def __rshift__(self, other): + return self.__wrapped__ >> other + + def __and__(self, other): + return self.__wrapped__ & other + + def __xor__(self, other): + return self.__wrapped__ ^ other + + def __or__(self, other): + return self.__wrapped__ | other + + def __radd__(self, other): + return other + self.__wrapped__ + + def __rsub__(self, other): + return other - self.__wrapped__ + + def __rmul__(self, other): + return other * self.__wrapped__ + + def __rdiv__(self, other): + return operator.div(other, self.__wrapped__) + + def __rtruediv__(self, other): + return operator.truediv(other, self.__wrapped__) + + def __rfloordiv__(self, other): + return other // self.__wrapped__ + + def __rmod__(self, other): + return other % self.__wrapped__ + + def __rdivmod__(self, other): + return divmod(other, self.__wrapped__) + + def __rpow__(self, other, *args): + return pow(other, self.__wrapped__, *args) + + def __rlshift__(self, other): + return other << self.__wrapped__ + + def __rrshift__(self, other): + return other >> self.__wrapped__ + + def __rand__(self, other): + return other & self.__wrapped__ + + def __rxor__(self, other): + return other ^ self.__wrapped__ + + def __ror__(self, other): + return other | self.__wrapped__ + + def __iadd__(self, other): + self.__wrapped__ += other + return self + + def __isub__(self, other): + self.__wrapped__ -= other + return self + + def __imul__(self, other): + self.__wrapped__ *= other + return self + + def __idiv__(self, other): + self.__wrapped__ = operator.idiv(self.__wrapped__, other) + return self + + def __itruediv__(self, other): + self.__wrapped__ = operator.itruediv(self.__wrapped__, other) + return self + + def __ifloordiv__(self, other): + self.__wrapped__ //= other + return self + + def __imod__(self, other): + self.__wrapped__ %= other + return self + + def __ipow__(self, other): + self.__wrapped__ **= other + return self + + def __ilshift__(self, other): + self.__wrapped__ <<= other + return self + + def __irshift__(self, other): + self.__wrapped__ >>= other + return self + + def __iand__(self, other): + self.__wrapped__ &= other + return self + + def __ixor__(self, other): + self.__wrapped__ ^= other + return self + + def __ior__(self, other): + self.__wrapped__ |= other + return self + + def __neg__(self): + return -self.__wrapped__ + + def __pos__(self): + return +self.__wrapped__ + + def __abs__(self): + return abs(self.__wrapped__) + + def __invert__(self): + return ~self.__wrapped__ + + def __int__(self): + return int(self.__wrapped__) + + if PY2: + def __long__(self): + return long(self.__wrapped__) # noqa + + def __float__(self): + return float(self.__wrapped__) + + def __oct__(self): + return oct(self.__wrapped__) + + def __hex__(self): + return hex(self.__wrapped__) + + def __index__(self): + return operator.index(self.__wrapped__) + + def __len__(self): + return len(self.__wrapped__) + + def __contains__(self, value): + return value in self.__wrapped__ + + def __getitem__(self, key): + return self.__wrapped__[key] + + def __setitem__(self, key, value): + self.__wrapped__[key] = value + + def __delitem__(self, key): + del self.__wrapped__[key] + + def __getslice__(self, i, j): + return self.__wrapped__[i:j] + + def __setslice__(self, i, j, value): + self.__wrapped__[i:j] = value + + def __delslice__(self, i, j): + del self.__wrapped__[i:j] + + def __enter__(self): + return self.__wrapped__.__enter__() + + def __exit__(self, *args, **kwargs): + return self.__wrapped__.__exit__(*args, **kwargs) + + def __iter__(self): + return iter(self.__wrapped__) + + def __call__(self, *args, **kwargs): + return self.__wrapped__(*args, **kwargs) + + def __reduce__(self): + return identity, (self.__wrapped__,) + + def __reduce_ex__(self, protocol): + return identity, (self.__wrapped__,) diff --git a/WebCrawler/venv/Lib/site-packages/lazy_object_proxy/utils.py b/WebCrawler/venv/Lib/site-packages/lazy_object_proxy/utils.py new file mode 100644 index 0000000..ceb3050 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/lazy_object_proxy/utils.py @@ -0,0 +1,13 @@ +def identity(obj): + return obj + + +class cached_property(object): + def __init__(self, func): + self.func = func + + def __get__(self, obj, cls): + if obj is None: + return self + value = obj.__dict__[self.func.__name__] = self.func(obj) + return value diff --git a/WebCrawler/venv/Lib/site-packages/lxml-4.6.2.dist-info/INSTALLER b/WebCrawler/venv/Lib/site-packages/lxml-4.6.2.dist-info/INSTALLER new file mode 100644 index 0000000..a1b589e --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/lxml-4.6.2.dist-info/INSTALLER @@ -0,0 +1 @@ +pip diff --git a/WebCrawler/venv/Lib/site-packages/lxml-4.6.2.dist-info/LICENSE.txt b/WebCrawler/venv/Lib/site-packages/lxml-4.6.2.dist-info/LICENSE.txt new file mode 100644 index 0000000..a76d0ed --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/lxml-4.6.2.dist-info/LICENSE.txt @@ -0,0 +1,29 @@ +Copyright (c) 2004 Infrae. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + + 3. Neither the name of Infrae nor the names of its contributors may + be used to endorse or promote products derived from this software + without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL INFRAE OR +CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/WebCrawler/venv/Lib/site-packages/lxml-4.6.2.dist-info/LICENSES.txt b/WebCrawler/venv/Lib/site-packages/lxml-4.6.2.dist-info/LICENSES.txt new file mode 100644 index 0000000..9f97c18 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/lxml-4.6.2.dist-info/LICENSES.txt @@ -0,0 +1,29 @@ +lxml is copyright Infrae and distributed under the BSD license (see +doc/licenses/BSD.txt), with the following exceptions: + +Some code, such a selftest.py, selftest2.py and +src/lxml/_elementpath.py are derived from ElementTree and +cElementTree. See doc/licenses/elementtree.txt for the license text. + +lxml.cssselect and lxml.html are copyright Ian Bicking and distributed +under the BSD license (see doc/licenses/BSD.txt). + +test.py, the test-runner script, is GPL and copyright Shuttleworth +Foundation. See doc/licenses/GPL.txt. It is believed the unchanged +inclusion of test.py to run the unit test suite falls under the +"aggregation" clause of the GPL and thus does not affect the license +of the rest of the package. + +The isoschematron implementation uses several XSL and RelaxNG resources: + * The (XML syntax) RelaxNG schema for schematron, copyright International + Organization for Standardization (see + src/lxml/isoschematron/resources/rng/iso-schematron.rng for the license + text) + * The skeleton iso-schematron-xlt1 pure-xslt schematron implementation + xsl stylesheets, copyright Rick Jelliffe and Academia Sinica Computing + Center, Taiwan (see the xsl files here for the license text: + src/lxml/isoschematron/resources/xsl/iso-schematron-xslt1/) + * The xsd/rng schema schematron extraction xsl transformations are unlicensed + and copyright the respective authors as noted (see + src/lxml/isoschematron/resources/xsl/RNG2Schtrn.xsl and + src/lxml/isoschematron/resources/xsl/XSD2Schtrn.xsl) diff --git a/WebCrawler/venv/Lib/site-packages/lxml-4.6.2.dist-info/METADATA b/WebCrawler/venv/Lib/site-packages/lxml-4.6.2.dist-info/METADATA new file mode 100644 index 0000000..e058233 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/lxml-4.6.2.dist-info/METADATA @@ -0,0 +1,82 @@ +Metadata-Version: 2.1 +Name: lxml +Version: 4.6.2 +Summary: Powerful and Pythonic XML processing library combining libxml2/libxslt with the ElementTree API. +Home-page: https://lxml.de/ +Author: lxml dev team +Author-email: lxml-dev@lxml.de +Maintainer: lxml dev team +Maintainer-email: lxml-dev@lxml.de +License: BSD +Platform: UNKNOWN +Classifier: Development Status :: 5 - Production/Stable +Classifier: Intended Audience :: Developers +Classifier: Intended Audience :: Information Technology +Classifier: License :: OSI Approved :: BSD License +Classifier: Programming Language :: Cython +Classifier: Programming Language :: Python :: 2 +Classifier: Programming Language :: Python :: 2.7 +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3.5 +Classifier: Programming Language :: Python :: 3.6 +Classifier: Programming Language :: Python :: 3.7 +Classifier: Programming Language :: Python :: 3.8 +Classifier: Programming Language :: Python :: 3.9 +Classifier: Programming Language :: C +Classifier: Operating System :: OS Independent +Classifier: Topic :: Text Processing :: Markup :: HTML +Classifier: Topic :: Text Processing :: Markup :: XML +Classifier: Topic :: Software Development :: Libraries :: Python Modules +Requires-Python: >=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, != 3.4.* +Provides-Extra: cssselect +Requires-Dist: cssselect (>=0.7) ; extra == 'cssselect' +Provides-Extra: html5 +Requires-Dist: html5lib ; extra == 'html5' +Provides-Extra: htmlsoup +Requires-Dist: BeautifulSoup4 ; extra == 'htmlsoup' +Provides-Extra: source +Requires-Dist: Cython (>=0.29.7) ; extra == 'source' + +lxml is a Pythonic, mature binding for the libxml2 and libxslt libraries. It +provides safe and convenient access to these libraries using the ElementTree +API. + +It extends the ElementTree API significantly to offer support for XPath, +RelaxNG, XML Schema, XSLT, C14N and much more. + +To contact the project, go to the `project home page +`_ or see our bug tracker at +https://launchpad.net/lxml + +In case you want to use the current in-development version of lxml, +you can get it from the github repository at +https://github.com/lxml/lxml . Note that this requires Cython to +build the sources, see the build instructions on the project home +page. To the same end, running ``easy_install lxml==dev`` will +install lxml from +https://github.com/lxml/lxml/tarball/master#egg=lxml-dev if you have +an appropriate version of Cython installed. + + +After an official release of a new stable series, bug fixes may become +available at +https://github.com/lxml/lxml/tree/lxml-4.6 . +Running ``easy_install lxml==4.6bugfix`` will install +the unreleased branch state from +https://github.com/lxml/lxml/tarball/lxml-4.6#egg=lxml-4.6bugfix +as soon as a maintenance branch has been established. Note that this +requires Cython to be installed at an appropriate version for the build. + +4.6.2 (2020-11-26) +================== + +Bugs fixed +---------- + +* A vulnerability (CVE-2020-27783) was discovered in the HTML Cleaner by Yaniv Nizry, + which allowed JavaScript to pass through. The cleaner now removes more sneaky + "style" content. + + + + diff --git a/WebCrawler/venv/Lib/site-packages/lxml-4.6.2.dist-info/RECORD b/WebCrawler/venv/Lib/site-packages/lxml-4.6.2.dist-info/RECORD new file mode 100644 index 0000000..a490468 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/lxml-4.6.2.dist-info/RECORD @@ -0,0 +1,168 @@ +lxml-4.6.2.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4 +lxml-4.6.2.dist-info/LICENSE.txt,sha256=ae20RcEzWoMS1MCScYR-mVbYTw2fck0SU0DMP612eyo,1488 +lxml-4.6.2.dist-info/LICENSES.txt,sha256=QdSd1AaqDhVIptXyGjDWv2OLPNlutyid00jYPtLkA5I,1514 +lxml-4.6.2.dist-info/METADATA,sha256=mswhtUj_lHJHMrYxgVyabWpNUTeBYVHQZIBo0VvANrY,3139 +lxml-4.6.2.dist-info/RECORD,, +lxml-4.6.2.dist-info/WHEEL,sha256=2Kg4PzfJLrLEnxRV1e1jZf0TVEjxVcXZXjp8WtjE4tI,105 +lxml-4.6.2.dist-info/top_level.txt,sha256=NjD988wqaKq512nshNdLt-uDxsjkp4Bh51m6N-dhUrk,5 +lxml/ElementInclude.py,sha256=PSLeZFvCa76WHJulPLxcZXJtCI2-4dK2CtqPRiYOAQg,8560 +lxml/__init__.py,sha256=el8BVyN5x2cxMqraECKDgzUWmLqzz9y_sjWuSmY7KWU,575 +lxml/__pycache__/ElementInclude.cpython-39.pyc,, +lxml/__pycache__/__init__.cpython-39.pyc,, +lxml/__pycache__/_elementpath.cpython-39.pyc,, +lxml/__pycache__/builder.cpython-39.pyc,, +lxml/__pycache__/cssselect.cpython-39.pyc,, +lxml/__pycache__/doctestcompare.cpython-39.pyc,, +lxml/__pycache__/pyclasslookup.cpython-39.pyc,, +lxml/__pycache__/sax.cpython-39.pyc,, +lxml/__pycache__/usedoctest.cpython-39.pyc,, +lxml/_elementpath.cp39-win_amd64.pyd,sha256=lG4uqJMTZcaD-0HxBeaDKE5Bgr-HbdckFoheXMJG5JM,144384 +lxml/_elementpath.py,sha256=wo6_CnGtKSkadI-krW8gbEZ1fVPTnIJINNwrdRfT_fw,10742 +lxml/builder.cp39-win_amd64.pyd,sha256=tNH8vOwyOc0DSsOg_jWjSxq6n3X99S_fcXVGy4q_qyA,81920 +lxml/builder.py,sha256=R5WsHI0bCOHYehskKXENvBkuarviL5rNJsmnh6cZGUA,7975 +lxml/cssselect.py,sha256=ADTqox2BUhZI_28K26Dnd-rPqvwL1A7KpXwDetXZLfA,3366 +lxml/doctestcompare.py,sha256=dAjqNzMGJuDsxY0xOXwOWzEsq7gSfQf-6uuxZZwaNXM,18339 +lxml/etree.cp39-win_amd64.pyd,sha256=jvzQQtTkU_m5csdVPZQlggSp8s40mqM7HL_SmK5cVM4,3859968 +lxml/etree.h,sha256=ZsIqmrdd4QENb9T3rgEFNvbvlELpylQGYiDspFWMVaU,8575 +lxml/etree_api.h,sha256=Jfz9HTaI52fPEGGgohrgLcjPimuhJV_vvWt9pnOoJvs,17467 +lxml/html/ElementSoup.py,sha256=s_dLobLMuKn2DhexR-iDXdZrMFg1RjLy1feHsIeZMpw,320 +lxml/html/__init__.py,sha256=16eBJ2AV4BDnoy56Qfkm6-81nY4fARw-wkL5lGoUmHQ,65090 +lxml/html/__pycache__/ElementSoup.cpython-39.pyc,, +lxml/html/__pycache__/__init__.cpython-39.pyc,, +lxml/html/__pycache__/_diffcommand.cpython-39.pyc,, +lxml/html/__pycache__/_html5builder.cpython-39.pyc,, +lxml/html/__pycache__/_setmixin.cpython-39.pyc,, +lxml/html/__pycache__/builder.cpython-39.pyc,, +lxml/html/__pycache__/clean.cpython-39.pyc,, +lxml/html/__pycache__/defs.cpython-39.pyc,, +lxml/html/__pycache__/diff.cpython-39.pyc,, +lxml/html/__pycache__/formfill.cpython-39.pyc,, +lxml/html/__pycache__/html5parser.cpython-39.pyc,, +lxml/html/__pycache__/soupparser.cpython-39.pyc,, +lxml/html/__pycache__/usedoctest.cpython-39.pyc,, +lxml/html/_diffcommand.py,sha256=7-tz3udrgg0unGPAI8pa_uN4e7vW0MmgOXE43kKPdw8,2121 +lxml/html/_html5builder.py,sha256=cASxN0Tks3_vqCA_sXa1oCx_McyRL6VpuRLA1T-B58o,3246 +lxml/html/_setmixin.py,sha256=uVCgBUC4SJ7N9GotmlKHrhH7R4Kk7wGU3u1WmEJKGeM,1184 +lxml/html/builder.py,sha256=aRgS-Ea9bli-muGX0iUQGKAe9D93P8BspQ2WPuiWJcU,4492 +lxml/html/clean.cp39-win_amd64.pyd,sha256=r66F5956lRA4hp45lwT6Y3l8gGK_tGhpZteADnxUzv4,189952 +lxml/html/clean.py,sha256=4rJhFRr9KG0wz-N0tpbLvFgrxhNRQxiYwRqZMM46jpQ,28038 +lxml/html/defs.py,sha256=MXAuq2-kkfQjOlhhoYaD51Nc7ESshnmIkglDIWYwuiw,4196 +lxml/html/diff.cp39-win_amd64.pyd,sha256=-muYadRiK3o1H6ohD_-53tk4PVCu0kFMhpOEVdboyB4,254464 +lxml/html/diff.py,sha256=F1A1vYO3dQdYfGDkI46zj5GN6UzaGMQt7ezqV5i324Q,30553 +lxml/html/formfill.py,sha256=9lnv7BnrQS0HOBY8ToeP1408xMN1wnltpsY-0CTGBpQ,9689 +lxml/html/html5parser.py,sha256=dnyC4cqHxywjZSzk0mu2L7THTZjxhg4yF4pncjusa_w,8634 +lxml/html/soupparser.py,sha256=tfdraMayPbMBCd2kGxUoTvNkhKUclfP3LmV9R85WKI4,10203 +lxml/html/usedoctest.py,sha256=tPlmVz4KK1GRKV5DJLrdVECeqsT9PlDzSqqTodVi5s0,249 +lxml/includes/__init__.pxd,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +lxml/includes/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +lxml/includes/__pycache__/__init__.cpython-39.pyc,, +lxml/includes/c14n.pxd,sha256=pGf910mVH9IdBb7r_aE-J59axIQcqFH4Sx_Tm0PA1m0,1123 +lxml/includes/config.pxd,sha256=H6Mrl8It21hzRI2hzMId9W48QqkYYkoLT4dniLNmdTw,96 +lxml/includes/dtdvalid.pxd,sha256=Rf2vRBbM4O1AOiIsUk_5M7pV3Dz309sS7Ccd2zGFHT0,671 +lxml/includes/etree_defs.h,sha256=e8Nlc-wG74xRyK3EqonkWx02LGzq7pl6bmLYzuWO9_Q,15559 +lxml/includes/etreepublic.pxd,sha256=3cdjIVlfkeZWYUav4y_T2uHwAo8yUCTlCvNLEvsZ_aI,10122 +lxml/includes/htmlparser.pxd,sha256=Va2qbs5zVokERn57HbDY__CiBQOoCS4uI9wEfCnT6zk,2868 +lxml/includes/libexslt/exslt.h,sha256=Z91WbHADa4o5-kJufstf6Y100qBTMqBiK1aZsxcp_Ao,3018 +lxml/includes/libexslt/exsltconfig.h,sha256=uUKX4TumO91Fg-Y2KB2Lwh-V6m4DB1DH1yO1z03YyWY,1291 +lxml/includes/libexslt/exsltexports.h,sha256=b5995w60ve9w6ZfdJCxtVnaP8kmY9chXldRmE_9DmG4,3383 +lxml/includes/libexslt/libexslt.h,sha256=EnT0rQzqljwg12CNnc-wd4IyWYO19IE5-nKxnCUefLQ,674 +lxml/includes/libxml/DOCBparser.h,sha256=QCzfpD-a5w-3T9fbCLhJI12H_iN5LVtOiWu3L21eyJo,3157 +lxml/includes/libxml/HTMLparser.h,sha256=3LY9MRXs5ZCMbF91gTQol33ceRpambWVY0ciEPu6otI,9410 +lxml/includes/libxml/HTMLtree.h,sha256=-i37-IqS_LU9jei_9yUi6Fs80ZErlrCGZWwaTHIg9p8,3646 +lxml/includes/libxml/SAX.h,sha256=EgEjbXP_0OtfSYEb6FHHnWmPEgnQYZ5IaX8Ehh7YvVg,4341 +lxml/includes/libxml/SAX2.h,sha256=9Z6vTKhysqsnLMv63aJTcyRLh6Qm8Base48qRZWMQ0M,4949 +lxml/includes/libxml/c14n.h,sha256=yJO-_m0HOgTfbBiv_j-P02LftZ8_ofzIpJEonBPyULY,3109 +lxml/includes/libxml/catalog.h,sha256=ugBmcsBjZefAsAE8vs7OwMZonYVBzDj2rNyWVF4YK3I,4905 +lxml/includes/libxml/chvalid.h,sha256=H9_NIeLyDhdKDZg9euK_5b5QmeXWWNq4KTENFkcUUtA,5159 +lxml/includes/libxml/debugXML.h,sha256=eatflryQfcmuAgmyz_f3JoSpTmvDA26mrcUjd7vwSLY,5152 +lxml/includes/libxml/dict.h,sha256=o36P53gHvahFTU4K-RMkeqy5oJzXza_akJ069CmdorE,1955 +lxml/includes/libxml/encoding.h,sha256=4D82UVBCn1biuxeAOarKDGqu5u-HOWZABy3-2Ns_IeQ,8301 +lxml/includes/libxml/entities.h,sha256=uJ8I3voqfwMEw8MQJveLtS0mOanjzzREreKrSs8_018,4712 +lxml/includes/libxml/globals.h,sha256=CHp7u46nJ-qE4N4tFVS6xRXnQgrnqEvKA0u2744F34M,14694 +lxml/includes/libxml/hash.h,sha256=yqN7eHBJ4GrDbMl-9D9DWWC_fOh7xDPF3OdVzKO-qdw,6492 +lxml/includes/libxml/list.h,sha256=QTym8NLQGYBe91ddyqhXW4fZIJBf13VHOC7T54RBzyw,3366 +lxml/includes/libxml/nanoftp.h,sha256=ZOKrFIQjW-h4d7gsG45h05GhpqdD9f9lCwgvP2D8l8Q,3762 +lxml/includes/libxml/nanohttp.h,sha256=LLgN_JcupVO24_Oki895yVKsJzFe5mcF2vH8Ekymq_I,2005 +lxml/includes/libxml/parser.h,sha256=fAvfRpely07ySdKS8MJRCwcyOoZpM-h5y6tPhXCZaNs,39717 +lxml/includes/libxml/parserInternals.h,sha256=2c5f6anmOJaYlqJhAky-mtn6CNBFFy1COvPXHtDkb2I,17418 +lxml/includes/libxml/pattern.h,sha256=FK85vxjka2kzTPZeCep57HMa7qc-I46WH3fOkUYOsWc,2586 +lxml/includes/libxml/relaxng.h,sha256=D1ufdqizDSPx8wOL_ZsjuIl--nK52cWMiZwIdEIbhSc,5996 +lxml/includes/libxml/schemasInternals.h,sha256=I1VSv5fO7wDHdgSoR2xrcUSGL1UEeOgabpnZT5wXayo,26243 +lxml/includes/libxml/schematron.h,sha256=lNOe9oFbhBqk1GVv4nJ4eZR3tieJVuw-gqbrmPtG3K4,4371 +lxml/includes/libxml/threads.h,sha256=xJjVOa3EFYx3INBFQ_-IHI4Xm6AwNBgAlLzfK8mNEJ8,1852 +lxml/includes/libxml/tree.h,sha256=80Hw_WvLrk4P3yruakmhoIzMXn0RLtsikDIGUk2n8Tc,38105 +lxml/includes/libxml/uri.h,sha256=nGU1MVe2zr7e3jReDYKCMPQkTkQRN-iNlLewkpP9cLo,2664 +lxml/includes/libxml/valid.h,sha256=__oCp9iK1nJc7s2iRZqtz8iHu6--9-XI_MWLXIBq_C0,13622 +lxml/includes/libxml/xinclude.h,sha256=iL_sqfaWIhThheHWBbEUQt59_sd-jnExhxaYeXxInqk,2967 +lxml/includes/libxml/xlink.h,sha256=hV-cvCxIJ_F1VdWcMYZ3w2jkGwWyLc9SZ8PdnUZxYNs,5040 +lxml/includes/libxml/xmlIO.h,sha256=-5MYRiCCuO81Y23ldZJU7Uh_my3U7yyg9rMfahcVans,10611 +lxml/includes/libxml/xmlautomata.h,sha256=rIJxLWBGZrfkFYDXdJZBoAhygp93vFBO_WI6nUCxBN4,3956 +lxml/includes/libxml/xmlerror.h,sha256=I7xskOqrTkctmlL76j2-_VMmMB_RPcXTglE1jHRXa5g,36808 +lxml/includes/libxml/xmlexports.h,sha256=xKNFpSDT-d53xTWeyClzXtsFusNiyo9l4Iwo4JNc7MM,3920 +lxml/includes/libxml/xmlmemory.h,sha256=Z6y0IQCIKXfVnYOPiGoFwc5Hq4rUdZ5AJwKJeav0TNw,5945 +lxml/includes/libxml/xmlmodule.h,sha256=c5vXusZj46FsM9UPHGt8g9gdPhTVOH1rJEFf2pJ8K4c,1170 +lxml/includes/libxml/xmlreader.h,sha256=mAAVz-ZYrOt56fzLds-ytusW7UqEGS5L9BGi5kzpTv4,12607 +lxml/includes/libxml/xmlregexp.h,sha256=UHbZDD3CZVn4NDiaHB64Aa-90pxzSsconMqneCp1ZDc,5458 +lxml/includes/libxml/xmlsave.h,sha256=8QlBKC6tuZ7lFQr_KJDTAxp4gmXtZWg-KGmLMkD7TCw,2337 +lxml/includes/libxml/xmlschemas.h,sha256=mvK9-OmM-FxfAPSpfkte4R-EWdkwvY6WFX6JmUeF3_U,7069 +lxml/includes/libxml/xmlschemastypes.h,sha256=alMLzZOds02Ksbv7K86owr0_IXjFPQ2Kk99rGhniZPE,4841 +lxml/includes/libxml/xmlstring.h,sha256=P_40FyEE40e_8h8Tlk9TSNze1zrJygFgsxkFydPnnPs,5511 +lxml/includes/libxml/xmlunicode.h,sha256=TpTZ8Bf313Rs9m-ww1cBCHwvW-uHj9tQ125iQfaDorU,9993 +lxml/includes/libxml/xmlversion.h,sha256=25QgUEOSP9i2QkQjjzbYfxImb0vy4VfuX3Fitfauub0,8634 +lxml/includes/libxml/xmlwriter.h,sha256=ddjdRh9PBH-FynmNgjkRg1JFBRzlZBPyyh7FX3JX1lY,21265 +lxml/includes/libxml/xpath.h,sha256=zo45KZIebdrOgK-Zu7kWQMm9yAs4pt_5b5QGnDApMc0,16398 +lxml/includes/libxml/xpathInternals.h,sha256=EWeEX1yCVHsUUeDxxOe-4jO25LBEDoM564CoPc1YNGU,19353 +lxml/includes/libxml/xpointer.h,sha256=6MQwhGyIy3PKp9yWaXRg0LAdL-bR2jjFNYUxUSkc6lk,3359 +lxml/includes/libxslt/attributes.h,sha256=vwJCNwOWLxNtoTh4Zqf9jhlxOjYJ-2e6KbaDEM7sEHY,930 +lxml/includes/libxslt/documents.h,sha256=kBihgH5pqRvFalhm_fOFHtJTFhTpBcm681yT5dxgwfw,2704 +lxml/includes/libxslt/extensions.h,sha256=bTyuv5_mj-RdPFhpT89LIM-iaMJ5dZtJFJkq0sbfaxE,6903 +lxml/includes/libxslt/extra.h,sha256=MqVdYuHZNw4wYQZdHs0mLY0xMjV-OoiZJeP7NG5iZgw,1641 +lxml/includes/libxslt/functions.h,sha256=LOR4F-xCPfST2QulovzKF2bQvan_Xw_jjaF5pqECDdM,2006 +lxml/includes/libxslt/imports.h,sha256=18kIjoGqdFXR63Ce3ZtzxsTiYV3XGKpchYakMUPDuUI,1840 +lxml/includes/libxslt/keys.h,sha256=16v25VEluS7jYhgg6gYFwVxgGMn-1ctnlhhWWT4RcBY,1155 +lxml/includes/libxslt/libxslt.h,sha256=2_zwp2lV_LzH4c9bitbn5vOyixfrnk1Yy26Cc2uEJ2I,860 +lxml/includes/libxslt/namespaces.h,sha256=VofSn2Kkn-a5JyRKCmY3jPp7amQy3n09vzy0KUQt4q0,1666 +lxml/includes/libxslt/numbersInternals.h,sha256=Eg5gYZ5p3h0_e5wyI61S-0E6_ArVJzv0yr63j6BU2fc,2019 +lxml/includes/libxslt/preproc.h,sha256=-19NH8t3q1NGsG6OsnctVHkk43tfNgbp1ei5glt_jNU,892 +lxml/includes/libxslt/security.h,sha256=fUD1cy_WxFCTvTNAF0WOQIU4p5CNWn1LHFyZJd-Fx5U,2652 +lxml/includes/libxslt/templates.h,sha256=bnt6Jqui6KU5pNUdMNPbQZkZ5d-VTWqC0TMGkOlVoIo,2268 +lxml/includes/libxslt/transform.h,sha256=UUHNbTEgGEVVaIhC86wi_CpffipfLob619bxiPpEwNI,6328 +lxml/includes/libxslt/trio.h,sha256=wTiet4_VXcF-NifQQHp_KFKIg4AYHI7WE3UcUzUk_z8,7199 +lxml/includes/libxslt/triodef.h,sha256=l1-U1p5j0OUAdi7sUXqL5mJ_TBbmzNJkHvWfTyLpx18,6692 +lxml/includes/libxslt/variables.h,sha256=09J1Jx8iDk1CXye24-3cYLYtI-ZaMwRh390jpLZ6TDo,3338 +lxml/includes/libxslt/win32config.h,sha256=RosX8sl9SQdWFn_1GLSX5OhQgoI78eeqfmEYQ0slbig,2929 +lxml/includes/libxslt/xslt.h,sha256=wmFx2Q31Pd8Iq2phAQpY9J3QQatb8lWg3gABtqKFgEw,1964 +lxml/includes/libxslt/xsltInternals.h,sha256=_kZYrVzE1EQlUJR8Y-QlVEUjuj4gCB0D7dxaNWXFkaw,57359 +lxml/includes/libxslt/xsltconfig.h,sha256=vFgdnmpfQSP8tRm6Bx5an9qrTSJOhun66DwZ7ChR3os,3825 +lxml/includes/libxslt/xsltexports.h,sha256=QLg1cfT3LW3m1iQnduDbstdg88cIBlPkaxMx3hlB7vE,3426 +lxml/includes/libxslt/xsltlocale.h,sha256=GL2phHwee_mJMaSh3K3oeRRuidcsTmomqe1blB7uil8,1396 +lxml/includes/libxslt/xsltutils.h,sha256=D2Gqop4cieJiOsQtLWTJezGea9kONcUFxeykLjGRI_E,8297 +lxml/includes/lxml-version.h,sha256=eyiHHbVw-mRjRj1HO_M62u2qYE8tu8-3BP6_43yufAk,74 +lxml/includes/relaxng.pxd,sha256=12yapjqDZLF_HTlcuSXSoQpPGK1NU7fj7gzS1EF8kZw,2669 +lxml/includes/schematron.pxd,sha256=5_PUpLHTzzYZ_d-8d2OjKLdwtLIdOm7C20HFUAX8hD4,1640 +lxml/includes/tree.pxd,sha256=dtnXNbEfxV3-5kOwWYkYoCE8HT5zvFVNdKFaIBuNXBc,20091 +lxml/includes/uri.pxd,sha256=5wPtpGU1JtdmpZMTzR8EswazihP3dxkns6Fgo9NWOt8,139 +lxml/includes/xinclude.pxd,sha256=onXD71LVdAbXjUj82_SDtSixNsNh8xbu6Nd9x0V3bmM,852 +lxml/includes/xmlerror.pxd,sha256=rV-nzG4TwgnNjuw7w68abmGDk_c8LGm4Y8f5oPw8OVc,57935 +lxml/includes/xmlparser.pxd,sha256=v-G3Y11eR4C1e2ImWLmGZpuTA90Y9N_uy-8H0-zwJY8,10852 +lxml/includes/xmlschema.pxd,sha256=yYQFrIKAQ_feenENV24X2AZyBIYGBltRDm9qB7CYMww,1696 +lxml/includes/xpath.pxd,sha256=tKYAcwpbSRq8qrsZ2ISVYvEaLnCV9GadNC5o_f8Ua_g,5794 +lxml/includes/xslt.pxd,sha256=qBU-0dLhIMQFv58I4q1XkBq9QJpJzKEAK9qBFPXBj_g,8341 +lxml/isoschematron/__init__.py,sha256=NUoI0bb87VB2XePzFcMtwUasrUAQACfb3Z9dPaMz6Fs,12399 +lxml/isoschematron/__pycache__/__init__.cpython-39.pyc,, +lxml/isoschematron/resources/rng/iso-schematron.rng,sha256=VsWxPyi3iViJDDbjJJw0wWkEHkLrz9zoCA8zJLor9N4,18337 +lxml/isoschematron/resources/xsl/RNG2Schtrn.xsl,sha256=ObebsB8Wt-d3uIA_U5NU85TpnQ3PxPX38TdOAqosMac,3172 +lxml/isoschematron/resources/xsl/XSD2Schtrn.xsl,sha256=QweRrIIM-zFcgg98GXA2CaWfIbgVE0XKEeYSfvv67A0,4563 +lxml/isoschematron/resources/xsl/iso-schematron-xslt1/iso_abstract_expand.xsl,sha256=xSZ_Ekq_I-62ZpiE5AqYYHwFW_qh855zt9V4_s7rbkY,11703 +lxml/isoschematron/resources/xsl/iso-schematron-xslt1/iso_dsdl_include.xsl,sha256=x42QJ-dxQ1waPzydsCoQnp2Xj15y53nW43O7BuoDRHk,39957 +lxml/isoschematron/resources/xsl/iso-schematron-xslt1/iso_schematron_message.xsl,sha256=Tr9BnO6pzjVWwhqJfm10UlvAy95EgfSCz2iMlrVGT6Q,2015 +lxml/isoschematron/resources/xsl/iso-schematron-xslt1/iso_schematron_skeleton_for_xslt1.xsl,sha256=ue8q_88X4e_jsJizo31GRNBxNhdxkEE9fY20oq0Iqwk,71764 +lxml/isoschematron/resources/xsl/iso-schematron-xslt1/iso_svrl_for_xslt1.xsl,sha256=BBAdsVSi5zAzeGepuN6gS1saQINDqITXKplmmj4dTWg,20382 +lxml/isoschematron/resources/xsl/iso-schematron-xslt1/readme.txt,sha256=OGLiFswuLJEW5EPYKOeoauuCJFEtVa6jyzBE1OcJI98,3310 +lxml/lxml.etree.h,sha256=f4vEV6rP4wPFoOdVPaNPl_SqbGg6AigSRcPad1jSz9Y,8799 +lxml/lxml.etree_api.h,sha256=Z6zBarmtBE567p4F3owHBNk6d1mIYIwjGAus21o_rMQ,17691 +lxml/objectify.cp39-win_amd64.pyd,sha256=1HrCOcsFNr2w5T6mKpp6YsN_SiXgge1IQlBefrgM_YE,1676800 +lxml/pyclasslookup.py,sha256=gLD1HM2HtITYYiGzjEOewSwbB7XkVx_NZv_quCt79Oc,92 +lxml/sax.cp39-win_amd64.pyd,sha256=hLyYQ_c12_F5zTPwFZKzX_blEJiTnseUOhBrd_kcr7g,125440 +lxml/sax.py,sha256=xcs8RDKFWnqCbZpYdDuJPLIUTbThrCYnX9arDhxlgOY,9396 +lxml/usedoctest.py,sha256=qRgZKQVcAZcl-zN0AIXVJnOsETUXz2nPXkxuzs1lGgk,230 diff --git a/WebCrawler/venv/Lib/site-packages/lxml-4.6.2.dist-info/WHEEL b/WebCrawler/venv/Lib/site-packages/lxml-4.6.2.dist-info/WHEEL new file mode 100644 index 0000000..358414f --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/lxml-4.6.2.dist-info/WHEEL @@ -0,0 +1,5 @@ +Wheel-Version: 1.0 +Generator: bdist_wheel (0.35.1) +Root-Is-Purelib: false +Tag: cp39-cp39-win_amd64 + diff --git a/WebCrawler/venv/Lib/site-packages/lxml-4.6.2.dist-info/top_level.txt b/WebCrawler/venv/Lib/site-packages/lxml-4.6.2.dist-info/top_level.txt new file mode 100644 index 0000000..ab90481 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/lxml-4.6.2.dist-info/top_level.txt @@ -0,0 +1 @@ +lxml diff --git a/WebCrawler/venv/Lib/site-packages/lxml/ElementInclude.py b/WebCrawler/venv/Lib/site-packages/lxml/ElementInclude.py new file mode 100644 index 0000000..2188433 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/lxml/ElementInclude.py @@ -0,0 +1,244 @@ +# +# ElementTree +# $Id: ElementInclude.py 1862 2004-06-18 07:31:02Z Fredrik $ +# +# limited xinclude support for element trees +# +# history: +# 2003-08-15 fl created +# 2003-11-14 fl fixed default loader +# +# Copyright (c) 2003-2004 by Fredrik Lundh. All rights reserved. +# +# fredrik@pythonware.com +# http://www.pythonware.com +# +# -------------------------------------------------------------------- +# The ElementTree toolkit is +# +# Copyright (c) 1999-2004 by Fredrik Lundh +# +# By obtaining, using, and/or copying this software and/or its +# associated documentation, you agree that you have read, understood, +# and will comply with the following terms and conditions: +# +# Permission to use, copy, modify, and distribute this software and +# its associated documentation for any purpose and without fee is +# hereby granted, provided that the above copyright notice appears in +# all copies, and that both that copyright notice and this permission +# notice appear in supporting documentation, and that the name of +# Secret Labs AB or the author not be used in advertising or publicity +# pertaining to distribution of the software without specific, written +# prior permission. +# +# SECRET LABS AB AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD +# TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANT- +# ABILITY AND FITNESS. IN NO EVENT SHALL SECRET LABS AB OR THE AUTHOR +# BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY +# DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, +# WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS +# ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE +# OF THIS SOFTWARE. +# -------------------------------------------------------------------- + +""" +Limited XInclude support for the ElementTree package. + +While lxml.etree has full support for XInclude (see +`etree.ElementTree.xinclude()`), this module provides a simpler, pure +Python, ElementTree compatible implementation that supports a simple +form of custom URL resolvers. +""" + +from lxml import etree +try: + from urlparse import urljoin + from urllib2 import urlopen +except ImportError: + # Python 3 + from urllib.parse import urljoin + from urllib.request import urlopen + +XINCLUDE = "{http://www.w3.org/2001/XInclude}" + +XINCLUDE_INCLUDE = XINCLUDE + "include" +XINCLUDE_FALLBACK = XINCLUDE + "fallback" +XINCLUDE_ITER_TAG = XINCLUDE + "*" + +# For security reasons, the inclusion depth is limited to this read-only value by default. +DEFAULT_MAX_INCLUSION_DEPTH = 6 + + +## +# Fatal include error. + +class FatalIncludeError(etree.LxmlSyntaxError): + pass + + +class LimitedRecursiveIncludeError(FatalIncludeError): + pass + + +## +# ET compatible default loader. +# This loader reads an included resource from disk. +# +# @param href Resource reference. +# @param parse Parse mode. Either "xml" or "text". +# @param encoding Optional text encoding. +# @return The expanded resource. If the parse mode is "xml", this +# is an ElementTree instance. If the parse mode is "text", this +# is a Unicode string. If the loader fails, it can return None +# or raise an IOError exception. +# @throws IOError If the loader fails to load the resource. + +def default_loader(href, parse, encoding=None): + file = open(href, 'rb') + if parse == "xml": + data = etree.parse(file).getroot() + else: + data = file.read() + if not encoding: + encoding = 'utf-8' + data = data.decode(encoding) + file.close() + return data + + +## +# Default loader used by lxml.etree - handles custom resolvers properly +# + +def _lxml_default_loader(href, parse, encoding=None, parser=None): + if parse == "xml": + data = etree.parse(href, parser).getroot() + else: + if "://" in href: + f = urlopen(href) + else: + f = open(href, 'rb') + data = f.read() + f.close() + if not encoding: + encoding = 'utf-8' + data = data.decode(encoding) + return data + + +## +# Wrapper for ET compatibility - drops the parser + +def _wrap_et_loader(loader): + def load(href, parse, encoding=None, parser=None): + return loader(href, parse, encoding) + return load + + +## +# Expand XInclude directives. +# +# @param elem Root element. +# @param loader Optional resource loader. If omitted, it defaults +# to {@link default_loader}. If given, it should be a callable +# that implements the same interface as default_loader. +# @param base_url The base URL of the original file, to resolve +# relative include file references. +# @param max_depth The maximum number of recursive inclusions. +# Limited to reduce the risk of malicious content explosion. +# Pass None to disable the limitation. +# @throws LimitedRecursiveIncludeError If the {@link max_depth} was exceeded. +# @throws FatalIncludeError If the function fails to include a given +# resource, or if the tree contains malformed XInclude elements. +# @throws IOError If the function fails to load a given resource. +# @returns the node or its replacement if it was an XInclude node + +def include(elem, loader=None, base_url=None, + max_depth=DEFAULT_MAX_INCLUSION_DEPTH): + if max_depth is None: + max_depth = -1 + elif max_depth < 0: + raise ValueError("expected non-negative depth or None for 'max_depth', got %r" % max_depth) + + if base_url is None: + if hasattr(elem, 'getroot'): + tree = elem + elem = elem.getroot() + else: + tree = elem.getroottree() + if hasattr(tree, 'docinfo'): + base_url = tree.docinfo.URL + elif hasattr(elem, 'getroot'): + elem = elem.getroot() + _include(elem, loader, base_url, max_depth) + + +def _include(elem, loader=None, base_url=None, + max_depth=DEFAULT_MAX_INCLUSION_DEPTH, _parent_hrefs=None): + if loader is not None: + load_include = _wrap_et_loader(loader) + else: + load_include = _lxml_default_loader + + if _parent_hrefs is None: + _parent_hrefs = set() + + parser = elem.getroottree().parser + + include_elements = list( + elem.iter(XINCLUDE_ITER_TAG)) + + for e in include_elements: + if e.tag == XINCLUDE_INCLUDE: + # process xinclude directive + href = urljoin(base_url, e.get("href")) + parse = e.get("parse", "xml") + parent = e.getparent() + if parse == "xml": + if href in _parent_hrefs: + raise FatalIncludeError( + "recursive include of %r detected" % href + ) + if max_depth == 0: + raise LimitedRecursiveIncludeError( + "maximum xinclude depth reached when including file %s" % href) + node = load_include(href, parse, parser=parser) + if node is None: + raise FatalIncludeError( + "cannot load %r as %r" % (href, parse) + ) + node = _include(node, loader, href, max_depth - 1, {href} | _parent_hrefs) + if e.tail: + node.tail = (node.tail or "") + e.tail + if parent is None: + return node # replaced the root node! + parent.replace(e, node) + elif parse == "text": + text = load_include(href, parse, encoding=e.get("encoding")) + if text is None: + raise FatalIncludeError( + "cannot load %r as %r" % (href, parse) + ) + predecessor = e.getprevious() + if predecessor is not None: + predecessor.tail = (predecessor.tail or "") + text + elif parent is None: + return text # replaced the root node! + else: + parent.text = (parent.text or "") + text + (e.tail or "") + parent.remove(e) + else: + raise FatalIncludeError( + "unknown parse type in xi:include tag (%r)" % parse + ) + elif e.tag == XINCLUDE_FALLBACK: + parent = e.getparent() + if parent is not None and parent.tag != XINCLUDE_INCLUDE: + raise FatalIncludeError( + "xi:fallback tag must be child of xi:include (%r)" % e.tag + ) + else: + raise FatalIncludeError( + "Invalid element found in XInclude namespace (%r)" % e.tag + ) + return elem diff --git a/WebCrawler/venv/Lib/site-packages/lxml/__init__.py b/WebCrawler/venv/Lib/site-packages/lxml/__init__.py new file mode 100644 index 0000000..ed50c4b --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/lxml/__init__.py @@ -0,0 +1,23 @@ +# this is a package + +__version__ = "4.6.2" + + +def get_include(): + """ + Returns a list of header include paths (for lxml itself, libxml2 + and libxslt) needed to compile C code against lxml if it was built + with statically linked libraries. + """ + import os + lxml_path = __path__[0] + include_path = os.path.join(lxml_path, 'includes') + includes = [include_path, lxml_path] + + for name in os.listdir(include_path): + path = os.path.join(include_path, name) + if os.path.isdir(path): + includes.append(path) + + return includes + diff --git a/WebCrawler/venv/Lib/site-packages/lxml/__pycache__/ElementInclude.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/lxml/__pycache__/ElementInclude.cpython-39.pyc new file mode 100644 index 0000000..536e7f6 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/lxml/__pycache__/ElementInclude.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/lxml/__pycache__/__init__.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/lxml/__pycache__/__init__.cpython-39.pyc new file mode 100644 index 0000000..334344d Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/lxml/__pycache__/__init__.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/lxml/__pycache__/_elementpath.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/lxml/__pycache__/_elementpath.cpython-39.pyc new file mode 100644 index 0000000..8960ce8 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/lxml/__pycache__/_elementpath.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/lxml/__pycache__/builder.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/lxml/__pycache__/builder.cpython-39.pyc new file mode 100644 index 0000000..b94d803 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/lxml/__pycache__/builder.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/lxml/__pycache__/cssselect.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/lxml/__pycache__/cssselect.cpython-39.pyc new file mode 100644 index 0000000..7f3b870 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/lxml/__pycache__/cssselect.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/lxml/__pycache__/doctestcompare.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/lxml/__pycache__/doctestcompare.cpython-39.pyc new file mode 100644 index 0000000..9ae133f Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/lxml/__pycache__/doctestcompare.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/lxml/__pycache__/pyclasslookup.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/lxml/__pycache__/pyclasslookup.cpython-39.pyc new file mode 100644 index 0000000..0db275d Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/lxml/__pycache__/pyclasslookup.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/lxml/__pycache__/sax.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/lxml/__pycache__/sax.cpython-39.pyc new file mode 100644 index 0000000..b027c37 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/lxml/__pycache__/sax.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/lxml/__pycache__/usedoctest.cpython-39.pyc b/WebCrawler/venv/Lib/site-packages/lxml/__pycache__/usedoctest.cpython-39.pyc new file mode 100644 index 0000000..6ce72ed Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/lxml/__pycache__/usedoctest.cpython-39.pyc differ diff --git a/WebCrawler/venv/Lib/site-packages/lxml/_elementpath.cp39-win_amd64.pyd b/WebCrawler/venv/Lib/site-packages/lxml/_elementpath.cp39-win_amd64.pyd new file mode 100644 index 0000000..e56bbca Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/lxml/_elementpath.cp39-win_amd64.pyd differ diff --git a/WebCrawler/venv/Lib/site-packages/lxml/_elementpath.py b/WebCrawler/venv/Lib/site-packages/lxml/_elementpath.py new file mode 100644 index 0000000..eabd81c --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/lxml/_elementpath.py @@ -0,0 +1,345 @@ +# cython: language_level=2 + +# +# ElementTree +# $Id: ElementPath.py 3375 2008-02-13 08:05:08Z fredrik $ +# +# limited xpath support for element trees +# +# history: +# 2003-05-23 fl created +# 2003-05-28 fl added support for // etc +# 2003-08-27 fl fixed parsing of periods in element names +# 2007-09-10 fl new selection engine +# 2007-09-12 fl fixed parent selector +# 2007-09-13 fl added iterfind; changed findall to return a list +# 2007-11-30 fl added namespaces support +# 2009-10-30 fl added child element value filter +# +# Copyright (c) 2003-2009 by Fredrik Lundh. All rights reserved. +# +# fredrik@pythonware.com +# http://www.pythonware.com +# +# -------------------------------------------------------------------- +# The ElementTree toolkit is +# +# Copyright (c) 1999-2009 by Fredrik Lundh +# +# By obtaining, using, and/or copying this software and/or its +# associated documentation, you agree that you have read, understood, +# and will comply with the following terms and conditions: +# +# Permission to use, copy, modify, and distribute this software and +# its associated documentation for any purpose and without fee is +# hereby granted, provided that the above copyright notice appears in +# all copies, and that both that copyright notice and this permission +# notice appear in supporting documentation, and that the name of +# Secret Labs AB or the author not be used in advertising or publicity +# pertaining to distribution of the software without specific, written +# prior permission. +# +# SECRET LABS AB AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD +# TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANT- +# ABILITY AND FITNESS. IN NO EVENT SHALL SECRET LABS AB OR THE AUTHOR +# BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY +# DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, +# WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS +# ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE +# OF THIS SOFTWARE. +# -------------------------------------------------------------------- + +## +# Implementation module for XPath support. There's usually no reason +# to import this module directly; the ElementTree does this for +# you, if needed. +## + +from __future__ import absolute_import + +import re + +xpath_tokenizer_re = re.compile( + "(" + "'[^']*'|\"[^\"]*\"|" + "::|" + "//?|" + r"\.\.|" + r"\(\)|" + r"[/.*:\[\]\(\)@=])|" + r"((?:\{[^}]+\})?[^/\[\]\(\)@=\s]+)|" + r"\s+" + ) + +def xpath_tokenizer(pattern, namespaces=None): + # ElementTree uses '', lxml used None originally. + default_namespace = (namespaces.get(None) or namespaces.get('')) if namespaces else None + parsing_attribute = False + for token in xpath_tokenizer_re.findall(pattern): + ttype, tag = token + if tag and tag[0] != "{": + if ":" in tag: + prefix, uri = tag.split(":", 1) + try: + if not namespaces: + raise KeyError + yield ttype, "{%s}%s" % (namespaces[prefix], uri) + except KeyError: + raise SyntaxError("prefix %r not found in prefix map" % prefix) + elif default_namespace and not parsing_attribute: + yield ttype, "{%s}%s" % (default_namespace, tag) + else: + yield token + parsing_attribute = False + else: + yield token + parsing_attribute = ttype == '@' + + +def prepare_child(next, token): + tag = token[1] + def select(result): + for elem in result: + for e in elem.iterchildren(tag): + yield e + return select + +def prepare_star(next, token): + def select(result): + for elem in result: + for e in elem.iterchildren('*'): + yield e + return select + +def prepare_self(next, token): + def select(result): + return result + return select + +def prepare_descendant(next, token): + token = next() + if token[0] == "*": + tag = "*" + elif not token[0]: + tag = token[1] + else: + raise SyntaxError("invalid descendant") + def select(result): + for elem in result: + for e in elem.iterdescendants(tag): + yield e + return select + +def prepare_parent(next, token): + def select(result): + for elem in result: + parent = elem.getparent() + if parent is not None: + yield parent + return select + +def prepare_predicate(next, token): + # FIXME: replace with real parser!!! refs: + # http://effbot.org/zone/simple-iterator-parser.htm + # http://javascript.crockford.com/tdop/tdop.html + signature = '' + predicate = [] + while 1: + token = next() + if token[0] == "]": + break + if token == ('', ''): + # ignore whitespace + continue + if token[0] and token[0][:1] in "'\"": + token = "'", token[0][1:-1] + signature += token[0] or "-" + predicate.append(token[1]) + + # use signature to determine predicate type + if signature == "@-": + # [@attribute] predicate + key = predicate[1] + def select(result): + for elem in result: + if elem.get(key) is not None: + yield elem + return select + if signature == "@-='": + # [@attribute='value'] + key = predicate[1] + value = predicate[-1] + def select(result): + for elem in result: + if elem.get(key) == value: + yield elem + return select + if signature == "-" and not re.match(r"-?\d+$", predicate[0]): + # [tag] + tag = predicate[0] + def select(result): + for elem in result: + for _ in elem.iterchildren(tag): + yield elem + break + return select + if signature == ".='" or (signature == "-='" and not re.match(r"-?\d+$", predicate[0])): + # [.='value'] or [tag='value'] + tag = predicate[0] + value = predicate[-1] + if tag: + def select(result): + for elem in result: + for e in elem.iterchildren(tag): + if "".join(e.itertext()) == value: + yield elem + break + else: + def select(result): + for elem in result: + if "".join(elem.itertext()) == value: + yield elem + return select + if signature == "-" or signature == "-()" or signature == "-()-": + # [index] or [last()] or [last()-index] + if signature == "-": + # [index] + index = int(predicate[0]) - 1 + if index < 0: + if index == -1: + raise SyntaxError( + "indices in path predicates are 1-based, not 0-based") + else: + raise SyntaxError("path index >= 1 expected") + else: + if predicate[0] != "last": + raise SyntaxError("unsupported function") + if signature == "-()-": + try: + index = int(predicate[2]) - 1 + except ValueError: + raise SyntaxError("unsupported expression") + else: + index = -1 + def select(result): + for elem in result: + parent = elem.getparent() + if parent is None: + continue + try: + # FIXME: what if the selector is "*" ? + elems = list(parent.iterchildren(elem.tag)) + if elems[index] is elem: + yield elem + except IndexError: + pass + return select + raise SyntaxError("invalid predicate") + +ops = { + "": prepare_child, + "*": prepare_star, + ".": prepare_self, + "..": prepare_parent, + "//": prepare_descendant, + "[": prepare_predicate, +} + + +# -------------------------------------------------------------------- + +_cache = {} + + +def _build_path_iterator(path, namespaces): + """compile selector pattern""" + if path[-1:] == "/": + path += "*" # implicit all (FIXME: keep this?) + + cache_key = (path,) + if namespaces: + # lxml originally used None for the default namespace but ElementTree uses the + # more convenient (all-strings-dict) empty string, so we support both here, + # preferring the more convenient '', as long as they aren't ambiguous. + if None in namespaces: + if '' in namespaces and namespaces[None] != namespaces['']: + raise ValueError("Ambiguous default namespace provided: %r versus %r" % ( + namespaces[None], namespaces[''])) + cache_key += (namespaces[None],) + tuple(sorted( + item for item in namespaces.items() if item[0] is not None)) + else: + cache_key += tuple(sorted(namespaces.items())) + + try: + return _cache[cache_key] + except KeyError: + pass + if len(_cache) > 100: + _cache.clear() + + if path[:1] == "/": + raise SyntaxError("cannot use absolute path on element") + stream = iter(xpath_tokenizer(path, namespaces)) + try: + _next = stream.next + except AttributeError: + # Python 3 + _next = stream.__next__ + try: + token = _next() + except StopIteration: + raise SyntaxError("empty path expression") + selector = [] + while 1: + try: + selector.append(ops[token[0]](_next, token)) + except StopIteration: + raise SyntaxError("invalid path") + try: + token = _next() + if token[0] == "/": + token = _next() + except StopIteration: + break + _cache[cache_key] = selector + return selector + + +## +# Iterate over the matching nodes + +def iterfind(elem, path, namespaces=None): + selector = _build_path_iterator(path, namespaces) + result = iter((elem,)) + for select in selector: + result = select(result) + return result + + +## +# Find first matching object. + +def find(elem, path, namespaces=None): + it = iterfind(elem, path, namespaces) + try: + return next(it) + except StopIteration: + return None + + +## +# Find all matching objects. + +def findall(elem, path, namespaces=None): + return list(iterfind(elem, path, namespaces)) + + +## +# Find text for first matching object. + +def findtext(elem, path, default=None, namespaces=None): + el = find(elem, path, namespaces) + if el is None: + return default + else: + return el.text or '' diff --git a/WebCrawler/venv/Lib/site-packages/lxml/builder.cp39-win_amd64.pyd b/WebCrawler/venv/Lib/site-packages/lxml/builder.cp39-win_amd64.pyd new file mode 100644 index 0000000..0617a15 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/lxml/builder.cp39-win_amd64.pyd differ diff --git a/WebCrawler/venv/Lib/site-packages/lxml/builder.py b/WebCrawler/venv/Lib/site-packages/lxml/builder.py new file mode 100644 index 0000000..a288845 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/lxml/builder.py @@ -0,0 +1,239 @@ +# cython: language_level=2 + +# +# Element generator factory by Fredrik Lundh. +# +# Source: +# http://online.effbot.org/2006_11_01_archive.htm#et-builder +# http://effbot.python-hosting.com/file/stuff/sandbox/elementlib/builder.py +# +# -------------------------------------------------------------------- +# The ElementTree toolkit is +# +# Copyright (c) 1999-2004 by Fredrik Lundh +# +# By obtaining, using, and/or copying this software and/or its +# associated documentation, you agree that you have read, understood, +# and will comply with the following terms and conditions: +# +# Permission to use, copy, modify, and distribute this software and +# its associated documentation for any purpose and without fee is +# hereby granted, provided that the above copyright notice appears in +# all copies, and that both that copyright notice and this permission +# notice appear in supporting documentation, and that the name of +# Secret Labs AB or the author not be used in advertising or publicity +# pertaining to distribution of the software without specific, written +# prior permission. +# +# SECRET LABS AB AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD +# TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANT- +# ABILITY AND FITNESS. IN NO EVENT SHALL SECRET LABS AB OR THE AUTHOR +# BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY +# DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, +# WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS +# ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE +# OF THIS SOFTWARE. +# -------------------------------------------------------------------- + +""" +The ``E`` Element factory for generating XML documents. +""" + +from __future__ import absolute_import + +import lxml.etree as ET + +from functools import partial + +try: + basestring +except NameError: + basestring = str + +try: + unicode +except NameError: + unicode = str + + +class ElementMaker(object): + """Element generator factory. + + Unlike the ordinary Element factory, the E factory allows you to pass in + more than just a tag and some optional attributes; you can also pass in + text and other elements. The text is added as either text or tail + attributes, and elements are inserted at the right spot. Some small + examples:: + + >>> from lxml import etree as ET + >>> from lxml.builder import E + + >>> ET.tostring(E("tag")) + '' + >>> ET.tostring(E("tag", "text")) + 'text' + >>> ET.tostring(E("tag", "text", key="value")) + 'text' + >>> ET.tostring(E("tag", E("subtag", "text"), "tail")) + 'texttail' + + For simple tags, the factory also allows you to write ``E.tag(...)`` instead + of ``E('tag', ...)``:: + + >>> ET.tostring(E.tag()) + '' + >>> ET.tostring(E.tag("text")) + 'text' + >>> ET.tostring(E.tag(E.subtag("text"), "tail")) + 'texttail' + + Here's a somewhat larger example; this shows how to generate HTML + documents, using a mix of prepared factory functions for inline elements, + nested ``E.tag`` calls, and embedded XHTML fragments:: + + # some common inline elements + A = E.a + I = E.i + B = E.b + + def CLASS(v): + # helper function, 'class' is a reserved word + return {'class': v} + + page = ( + E.html( + E.head( + E.title("This is a sample document") + ), + E.body( + E.h1("Hello!", CLASS("title")), + E.p("This is a paragraph with ", B("bold"), " text in it!"), + E.p("This is another paragraph, with a ", + A("link", href="http://www.python.org"), "."), + E.p("Here are some reserved characters: ."), + ET.XML("

And finally, here is an embedded XHTML fragment.

"), + ) + ) + ) + + print ET.tostring(page) + + Here's a prettyprinted version of the output from the above script:: + + + + This is a sample document + + +

Hello!

+

This is a paragraph with bold text in it!

+

This is another paragraph, with link.

+

Here are some reserved characters: <spam&egg>.

+

And finally, here is an embedded XHTML fragment.

+ + + + For namespace support, you can pass a namespace map (``nsmap``) + and/or a specific target ``namespace`` to the ElementMaker class:: + + >>> E = ElementMaker(namespace="http://my.ns/") + >>> print(ET.tostring( E.test )) + + + >>> E = ElementMaker(namespace="http://my.ns/", nsmap={'p':'http://my.ns/'}) + >>> print(ET.tostring( E.test )) + + """ + + def __init__(self, typemap=None, + namespace=None, nsmap=None, makeelement=None): + if namespace is not None: + self._namespace = '{' + namespace + '}' + else: + self._namespace = None + + if nsmap: + self._nsmap = dict(nsmap) + else: + self._nsmap = None + + if makeelement is not None: + assert callable(makeelement) + self._makeelement = makeelement + else: + self._makeelement = ET.Element + + # initialize type map for this element factory + + if typemap: + typemap = dict(typemap) + else: + typemap = {} + + def add_text(elem, item): + try: + elem[-1].tail = (elem[-1].tail or "") + item + except IndexError: + elem.text = (elem.text or "") + item + + def add_cdata(elem, cdata): + if elem.text: + raise ValueError("Can't add a CDATA section. Element already has some text: %r" % elem.text) + elem.text = cdata + + if str not in typemap: + typemap[str] = add_text + if unicode not in typemap: + typemap[unicode] = add_text + if ET.CDATA not in typemap: + typemap[ET.CDATA] = add_cdata + + def add_dict(elem, item): + attrib = elem.attrib + for k, v in item.items(): + if isinstance(v, basestring): + attrib[k] = v + else: + attrib[k] = typemap[type(v)](None, v) + if dict not in typemap: + typemap[dict] = add_dict + + self._typemap = typemap + + def __call__(self, tag, *children, **attrib): + typemap = self._typemap + + if self._namespace is not None and tag[0] != '{': + tag = self._namespace + tag + elem = self._makeelement(tag, nsmap=self._nsmap) + if attrib: + typemap[dict](elem, attrib) + + for item in children: + if callable(item): + item = item() + t = typemap.get(type(item)) + if t is None: + if ET.iselement(item): + elem.append(item) + continue + for basetype in type(item).__mro__: + # See if the typemap knows of any of this type's bases. + t = typemap.get(basetype) + if t is not None: + break + else: + raise TypeError("bad argument type: %s(%r)" % + (type(item).__name__, item)) + v = t(elem, item) + if v: + typemap.get(type(v))(elem, v) + + return elem + + def __getattr__(self, tag): + return partial(self, tag) + + +# create factory object +E = ElementMaker() diff --git a/WebCrawler/venv/Lib/site-packages/lxml/cssselect.py b/WebCrawler/venv/Lib/site-packages/lxml/cssselect.py new file mode 100644 index 0000000..586a142 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/lxml/cssselect.py @@ -0,0 +1,102 @@ +"""CSS Selectors based on XPath. + +This module supports selecting XML/HTML tags based on CSS selectors. +See the `CSSSelector` class for details. + +This is a thin wrapper around cssselect 0.7 or later. +""" + +from __future__ import absolute_import + +from . import etree +try: + import cssselect as external_cssselect +except ImportError: + raise ImportError( + 'cssselect does not seem to be installed. ' + 'See http://packages.python.org/cssselect/') + + +SelectorSyntaxError = external_cssselect.SelectorSyntaxError +ExpressionError = external_cssselect.ExpressionError +SelectorError = external_cssselect.SelectorError + + +__all__ = ['SelectorSyntaxError', 'ExpressionError', 'SelectorError', + 'CSSSelector'] + + +class LxmlTranslator(external_cssselect.GenericTranslator): + """ + A custom CSS selector to XPath translator with lxml-specific extensions. + """ + def xpath_contains_function(self, xpath, function): + # Defined there, removed in later drafts: + # http://www.w3.org/TR/2001/CR-css3-selectors-20011113/#content-selectors + if function.argument_types() not in (['STRING'], ['IDENT']): + raise ExpressionError( + "Expected a single string or ident for :contains(), got %r" + % function.arguments) + value = function.arguments[0].value + return xpath.add_condition( + 'contains(__lxml_internal_css:lower-case(string(.)), %s)' + % self.xpath_literal(value.lower())) + + +class LxmlHTMLTranslator(LxmlTranslator, external_cssselect.HTMLTranslator): + """ + lxml extensions + HTML support. + """ + + +def _make_lower_case(context, s): + return s.lower() + +ns = etree.FunctionNamespace('http://codespeak.net/lxml/css/') +ns.prefix = '__lxml_internal_css' +ns['lower-case'] = _make_lower_case + + +class CSSSelector(etree.XPath): + """A CSS selector. + + Usage:: + + >>> from lxml import etree, cssselect + >>> select = cssselect.CSSSelector("a tag > child") + + >>> root = etree.XML("TEXT") + >>> [ el.tag for el in select(root) ] + ['child'] + + To use CSS namespaces, you need to pass a prefix-to-namespace + mapping as ``namespaces`` keyword argument:: + + >>> rdfns = 'http://www.w3.org/1999/02/22-rdf-syntax-ns#' + >>> select_ns = cssselect.CSSSelector('root > rdf|Description', + ... namespaces={'rdf': rdfns}) + + >>> rdf = etree.XML(( + ... '' + ... 'blah' + ... '') % rdfns) + >>> [(el.tag, el.text) for el in select_ns(rdf)] + [('{http://www.w3.org/1999/02/22-rdf-syntax-ns#}Description', 'blah')] + + """ + def __init__(self, css, namespaces=None, translator='xml'): + if translator == 'xml': + translator = LxmlTranslator() + elif translator == 'html': + translator = LxmlHTMLTranslator() + elif translator == 'xhtml': + translator = LxmlHTMLTranslator(xhtml=True) + path = translator.css_to_xpath(css) + etree.XPath.__init__(self, path, namespaces=namespaces) + self.css = css + + def __repr__(self): + return '<%s %s for %r>' % ( + self.__class__.__name__, + hex(abs(id(self)))[2:], + self.css) diff --git a/WebCrawler/venv/Lib/site-packages/lxml/doctestcompare.py b/WebCrawler/venv/Lib/site-packages/lxml/doctestcompare.py new file mode 100644 index 0000000..1b0daa4 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/lxml/doctestcompare.py @@ -0,0 +1,507 @@ +""" +lxml-based doctest output comparison. + +Note: normally, you should just import the `lxml.usedoctest` and +`lxml.html.usedoctest` modules from within a doctest, instead of this +one:: + + >>> import lxml.usedoctest # for XML output + + >>> import lxml.html.usedoctest # for HTML output + +To use this module directly, you must call ``lxmldoctest.install()``, +which will cause doctest to use this in all subsequent calls. + +This changes the way output is checked and comparisons are made for +XML or HTML-like content. + +XML or HTML content is noticed because the example starts with ``<`` +(it's HTML if it starts with ```` or include an ``any`` +attribute in the tag. An ``any`` tag matches any tag, while the +attribute matches any and all attributes. + +When a match fails, the reformatted example and gotten text is +displayed (indented), and a rough diff-like output is given. Anything +marked with ``+`` is in the output but wasn't supposed to be, and +similarly ``-`` means its in the example but wasn't in the output. + +You can disable parsing on one line with ``# doctest:+NOPARSE_MARKUP`` +""" + +from lxml import etree +import sys +import re +import doctest +try: + from html import escape as html_escape +except ImportError: + from cgi import escape as html_escape + +__all__ = ['PARSE_HTML', 'PARSE_XML', 'NOPARSE_MARKUP', 'LXMLOutputChecker', + 'LHTMLOutputChecker', 'install', 'temp_install'] + +try: + _basestring = basestring +except NameError: + _basestring = (str, bytes) + +_IS_PYTHON_3 = sys.version_info[0] >= 3 + +PARSE_HTML = doctest.register_optionflag('PARSE_HTML') +PARSE_XML = doctest.register_optionflag('PARSE_XML') +NOPARSE_MARKUP = doctest.register_optionflag('NOPARSE_MARKUP') + +OutputChecker = doctest.OutputChecker + +def strip(v): + if v is None: + return None + else: + return v.strip() + +def norm_whitespace(v): + return _norm_whitespace_re.sub(' ', v) + +_html_parser = etree.HTMLParser(recover=False, remove_blank_text=True) + +def html_fromstring(html): + return etree.fromstring(html, _html_parser) + +# We use this to distinguish repr()s from elements: +_repr_re = re.compile(r'^<[^>]+ (at|object) ') +_norm_whitespace_re = re.compile(r'[ \t\n][ \t\n]+') + +class LXMLOutputChecker(OutputChecker): + + empty_tags = ( + 'param', 'img', 'area', 'br', 'basefont', 'input', + 'base', 'meta', 'link', 'col') + + def get_default_parser(self): + return etree.XML + + def check_output(self, want, got, optionflags): + alt_self = getattr(self, '_temp_override_self', None) + if alt_self is not None: + super_method = self._temp_call_super_check_output + self = alt_self + else: + super_method = OutputChecker.check_output + parser = self.get_parser(want, got, optionflags) + if not parser: + return super_method( + self, want, got, optionflags) + try: + want_doc = parser(want) + except etree.XMLSyntaxError: + return False + try: + got_doc = parser(got) + except etree.XMLSyntaxError: + return False + return self.compare_docs(want_doc, got_doc) + + def get_parser(self, want, got, optionflags): + parser = None + if NOPARSE_MARKUP & optionflags: + return None + if PARSE_HTML & optionflags: + parser = html_fromstring + elif PARSE_XML & optionflags: + parser = etree.XML + elif (want.strip().lower().startswith('' % el.tag + return '<%s %s>' % (el.tag, ' '.join(attrs)) + + def format_end_tag(self, el): + if isinstance(el, etree.CommentBase): + # FIXME: probably PIs should be handled specially too? + return '-->' + return '' % el.tag + + def collect_diff(self, want, got, html, indent): + parts = [] + if not len(want) and not len(got): + parts.append(' '*indent) + parts.append(self.collect_diff_tag(want, got)) + if not self.html_empty_tag(got, html): + parts.append(self.collect_diff_text(want.text, got.text)) + parts.append(self.collect_diff_end_tag(want, got)) + parts.append(self.collect_diff_text(want.tail, got.tail)) + parts.append('\n') + return ''.join(parts) + parts.append(' '*indent) + parts.append(self.collect_diff_tag(want, got)) + parts.append('\n') + if strip(want.text) or strip(got.text): + parts.append(' '*indent) + parts.append(self.collect_diff_text(want.text, got.text)) + parts.append('\n') + want_children = list(want) + got_children = list(got) + while want_children or got_children: + if not want_children: + parts.append(self.format_doc(got_children.pop(0), html, indent+2, '+')) + continue + if not got_children: + parts.append(self.format_doc(want_children.pop(0), html, indent+2, '-')) + continue + parts.append(self.collect_diff( + want_children.pop(0), got_children.pop(0), html, indent+2)) + parts.append(' '*indent) + parts.append(self.collect_diff_end_tag(want, got)) + parts.append('\n') + if strip(want.tail) or strip(got.tail): + parts.append(' '*indent) + parts.append(self.collect_diff_text(want.tail, got.tail)) + parts.append('\n') + return ''.join(parts) + + def collect_diff_tag(self, want, got): + if not self.tag_compare(want.tag, got.tag): + tag = '%s (got: %s)' % (want.tag, got.tag) + else: + tag = got.tag + attrs = [] + any = want.tag == 'any' or 'any' in want.attrib + for name, value in sorted(got.attrib.items()): + if name not in want.attrib and not any: + attrs.append('+%s="%s"' % (name, self.format_text(value, False))) + else: + if name in want.attrib: + text = self.collect_diff_text(want.attrib[name], value, False) + else: + text = self.format_text(value, False) + attrs.append('%s="%s"' % (name, text)) + if not any: + for name, value in sorted(want.attrib.items()): + if name in got.attrib: + continue + attrs.append('-%s="%s"' % (name, self.format_text(value, False))) + if attrs: + tag = '<%s %s>' % (tag, ' '.join(attrs)) + else: + tag = '<%s>' % tag + return tag + + def collect_diff_end_tag(self, want, got): + if want.tag != got.tag: + tag = '%s (got: %s)' % (want.tag, got.tag) + else: + tag = got.tag + return '' % tag + + def collect_diff_text(self, want, got, strip=True): + if self.text_compare(want, got, strip): + if not got: + return '' + return self.format_text(got, strip) + text = '%s (got: %s)' % (want, got) + return self.format_text(text, strip) + +class LHTMLOutputChecker(LXMLOutputChecker): + def get_default_parser(self): + return html_fromstring + +def install(html=False): + """ + Install doctestcompare for all future doctests. + + If html is true, then by default the HTML parser will be used; + otherwise the XML parser is used. + """ + if html: + doctest.OutputChecker = LHTMLOutputChecker + else: + doctest.OutputChecker = LXMLOutputChecker + +def temp_install(html=False, del_module=None): + """ + Use this *inside* a doctest to enable this checker for this + doctest only. + + If html is true, then by default the HTML parser will be used; + otherwise the XML parser is used. + """ + if html: + Checker = LHTMLOutputChecker + else: + Checker = LXMLOutputChecker + frame = _find_doctest_frame() + dt_self = frame.f_locals['self'] + checker = Checker() + old_checker = dt_self._checker + dt_self._checker = checker + # The unfortunate thing is that there is a local variable 'check' + # in the function that runs the doctests, that is a bound method + # into the output checker. We have to update that. We can't + # modify the frame, so we have to modify the object in place. The + # only way to do this is to actually change the func_code + # attribute of the method. We change it, and then wait for + # __record_outcome to be run, which signals the end of the __run + # method, at which point we restore the previous check_output + # implementation. + if _IS_PYTHON_3: + check_func = frame.f_locals['check'].__func__ + checker_check_func = checker.check_output.__func__ + else: + check_func = frame.f_locals['check'].im_func + checker_check_func = checker.check_output.im_func + # Because we can't patch up func_globals, this is the only global + # in check_output that we care about: + doctest.etree = etree + _RestoreChecker(dt_self, old_checker, checker, + check_func, checker_check_func, + del_module) + +class _RestoreChecker(object): + def __init__(self, dt_self, old_checker, new_checker, check_func, clone_func, + del_module): + self.dt_self = dt_self + self.checker = old_checker + self.checker._temp_call_super_check_output = self.call_super + self.checker._temp_override_self = new_checker + self.check_func = check_func + self.clone_func = clone_func + self.del_module = del_module + self.install_clone() + self.install_dt_self() + def install_clone(self): + if _IS_PYTHON_3: + self.func_code = self.check_func.__code__ + self.func_globals = self.check_func.__globals__ + self.check_func.__code__ = self.clone_func.__code__ + else: + self.func_code = self.check_func.func_code + self.func_globals = self.check_func.func_globals + self.check_func.func_code = self.clone_func.func_code + def uninstall_clone(self): + if _IS_PYTHON_3: + self.check_func.__code__ = self.func_code + else: + self.check_func.func_code = self.func_code + def install_dt_self(self): + self.prev_func = self.dt_self._DocTestRunner__record_outcome + self.dt_self._DocTestRunner__record_outcome = self + def uninstall_dt_self(self): + self.dt_self._DocTestRunner__record_outcome = self.prev_func + def uninstall_module(self): + if self.del_module: + import sys + del sys.modules[self.del_module] + if '.' in self.del_module: + package, module = self.del_module.rsplit('.', 1) + package_mod = sys.modules[package] + delattr(package_mod, module) + def __call__(self, *args, **kw): + self.uninstall_clone() + self.uninstall_dt_self() + del self.checker._temp_override_self + del self.checker._temp_call_super_check_output + result = self.prev_func(*args, **kw) + self.uninstall_module() + return result + def call_super(self, *args, **kw): + self.uninstall_clone() + try: + return self.check_func(*args, **kw) + finally: + self.install_clone() + +def _find_doctest_frame(): + import sys + frame = sys._getframe(1) + while frame: + l = frame.f_locals + if 'BOOM' in l: + # Sign of doctest + return frame + frame = frame.f_back + raise LookupError( + "Could not find doctest (only use this function *inside* a doctest)") + +__test__ = { + 'basic': ''' + >>> temp_install() + >>> print """stuff""" + ... + >>> print """""" + + + + >>> print """blahblahblah""" # doctest: +NOPARSE_MARKUP, +ELLIPSIS + ...foo /> + '''} + +if __name__ == '__main__': + import doctest + doctest.testmod() + + diff --git a/WebCrawler/venv/Lib/site-packages/lxml/etree.cp39-win_amd64.pyd b/WebCrawler/venv/Lib/site-packages/lxml/etree.cp39-win_amd64.pyd new file mode 100644 index 0000000..34cfdd2 Binary files /dev/null and b/WebCrawler/venv/Lib/site-packages/lxml/etree.cp39-win_amd64.pyd differ diff --git a/WebCrawler/venv/Lib/site-packages/lxml/etree.h b/WebCrawler/venv/Lib/site-packages/lxml/etree.h new file mode 100644 index 0000000..1d19136 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/lxml/etree.h @@ -0,0 +1,224 @@ +/* Generated by Cython 0.29.21 */ + +#ifndef __PYX_HAVE__lxml__etree +#define __PYX_HAVE__lxml__etree + +#include "Python.h" +struct LxmlDocument; +struct LxmlElement; +struct LxmlElementTree; +struct LxmlElementTagMatcher; +struct LxmlElementIterator; +struct LxmlElementBase; +struct LxmlElementClassLookup; +struct LxmlFallbackElementClassLookup; + +/* "lxml/etree.pyx":322 + * + * # type of a function that steps from node to node + * ctypedef public xmlNode* (*_node_to_node_function)(xmlNode*) # <<<<<<<<<<<<<< + * + * + */ +typedef xmlNode *(*_node_to_node_function)(xmlNode *); + +/* "lxml/etree.pyx":338 + * @cython.final + * @cython.freelist(8) + * cdef public class _Document [ type LxmlDocumentType, object LxmlDocument ]: # <<<<<<<<<<<<<< + * u"""Internal base class to reference a libxml document. + * + */ +struct LxmlDocument { + PyObject_HEAD + struct __pyx_vtabstruct_4lxml_5etree__Document *__pyx_vtab; + int _ns_counter; + PyObject *_prefix_tail; + xmlDoc *_c_doc; + struct __pyx_obj_4lxml_5etree__BaseParser *_parser; +}; + +/* "lxml/etree.pyx":687 + * + * @cython.no_gc_clear + * cdef public class _Element [ type LxmlElementType, object LxmlElement ]: # <<<<<<<<<<<<<< + * u"""Element class. + * + */ +struct LxmlElement { + PyObject_HEAD + struct LxmlDocument *_doc; + xmlNode *_c_node; + PyObject *_tag; +}; + +/* "lxml/etree.pyx":1854 + * + * + * cdef public class _ElementTree [ type LxmlElementTreeType, # <<<<<<<<<<<<<< + * object LxmlElementTree ]: + * cdef _Document _doc + */ +struct LxmlElementTree { + PyObject_HEAD + struct __pyx_vtabstruct_4lxml_5etree__ElementTree *__pyx_vtab; + struct LxmlDocument *_doc; + struct LxmlElement *_context_node; +}; + +/* "lxml/etree.pyx":2598 + * + * + * cdef public class _ElementTagMatcher [ object LxmlElementTagMatcher, # <<<<<<<<<<<<<< + * type LxmlElementTagMatcherType ]: + * """ + */ +struct LxmlElementTagMatcher { + PyObject_HEAD + struct __pyx_vtabstruct_4lxml_5etree__ElementTagMatcher *__pyx_vtab; + PyObject *_pystrings; + int _node_type; + char *_href; + char *_name; +}; + +/* "lxml/etree.pyx":2629 + * self._name = NULL + * + * cdef public class _ElementIterator(_ElementTagMatcher) [ # <<<<<<<<<<<<<< + * object LxmlElementIterator, type LxmlElementIteratorType ]: + * """ + */ +struct LxmlElementIterator { + struct LxmlElementTagMatcher __pyx_base; + struct LxmlElement *_node; + _node_to_node_function _next_element; +}; + +/* "src/lxml/classlookup.pxi":6 + * # Custom Element classes + * + * cdef public class ElementBase(_Element) [ type LxmlElementBaseType, # <<<<<<<<<<<<<< + * object LxmlElementBase ]: + * u"""ElementBase(*children, attrib=None, nsmap=None, **_extra) + */ +struct LxmlElementBase { + struct LxmlElement __pyx_base; +}; + +/* "src/lxml/classlookup.pxi":210 + * # Element class lookup + * + * ctypedef public object (*_element_class_lookup_function)(object, _Document, xmlNode*) # <<<<<<<<<<<<<< + * + * # class to store element class lookup functions + */ +typedef PyObject *(*_element_class_lookup_function)(PyObject *, struct LxmlDocument *, xmlNode *); + +/* "src/lxml/classlookup.pxi":213 + * + * # class to store element class lookup functions + * cdef public class ElementClassLookup [ type LxmlElementClassLookupType, # <<<<<<<<<<<<<< + * object LxmlElementClassLookup ]: + * u"""ElementClassLookup(self) + */ +struct LxmlElementClassLookup { + PyObject_HEAD + _element_class_lookup_function _lookup_function; +}; + +/* "src/lxml/classlookup.pxi":221 + * + * + * cdef public class FallbackElementClassLookup(ElementClassLookup) \ # <<<<<<<<<<<<<< + * [ type LxmlFallbackElementClassLookupType, + * object LxmlFallbackElementClassLookup ]: + */ +struct LxmlFallbackElementClassLookup { + struct LxmlElementClassLookup __pyx_base; + struct __pyx_vtabstruct_4lxml_5etree_FallbackElementClassLookup *__pyx_vtab; + struct LxmlElementClassLookup *fallback; + _element_class_lookup_function _fallback_function; +}; + +#ifndef __PYX_HAVE_API__lxml__etree + +#ifndef __PYX_EXTERN_C + #ifdef __cplusplus + #define __PYX_EXTERN_C extern "C" + #else + #define __PYX_EXTERN_C extern + #endif +#endif + +#ifndef DL_IMPORT + #define DL_IMPORT(_T) _T +#endif + +__PYX_EXTERN_C DL_IMPORT(PyTypeObject) LxmlDocumentType; +__PYX_EXTERN_C DL_IMPORT(PyTypeObject) LxmlElementType; +__PYX_EXTERN_C DL_IMPORT(PyTypeObject) LxmlElementTreeType; +__PYX_EXTERN_C DL_IMPORT(PyTypeObject) LxmlElementTagMatcherType; +__PYX_EXTERN_C DL_IMPORT(PyTypeObject) LxmlElementIteratorType; +__PYX_EXTERN_C DL_IMPORT(PyTypeObject) LxmlElementBaseType; +__PYX_EXTERN_C DL_IMPORT(PyTypeObject) LxmlElementClassLookupType; +__PYX_EXTERN_C DL_IMPORT(PyTypeObject) LxmlFallbackElementClassLookupType; + +__PYX_EXTERN_C struct LxmlElement *deepcopyNodeToDocument(struct LxmlDocument *, xmlNode *); +__PYX_EXTERN_C struct LxmlElementTree *elementTreeFactory(struct LxmlElement *); +__PYX_EXTERN_C struct LxmlElementTree *newElementTree(struct LxmlElement *, PyObject *); +__PYX_EXTERN_C struct LxmlElementTree *adoptExternalDocument(xmlDoc *, PyObject *, int); +__PYX_EXTERN_C struct LxmlElement *elementFactory(struct LxmlDocument *, xmlNode *); +__PYX_EXTERN_C struct LxmlElement *makeElement(PyObject *, struct LxmlDocument *, PyObject *, PyObject *, PyObject *, PyObject *, PyObject *); +__PYX_EXTERN_C struct LxmlElement *makeSubElement(struct LxmlElement *, PyObject *, PyObject *, PyObject *, PyObject *, PyObject *); +__PYX_EXTERN_C void setElementClassLookupFunction(_element_class_lookup_function, PyObject *); +__PYX_EXTERN_C PyObject *lookupDefaultElementClass(PyObject *, PyObject *, xmlNode *); +__PYX_EXTERN_C PyObject *lookupNamespaceElementClass(PyObject *, PyObject *, xmlNode *); +__PYX_EXTERN_C PyObject *callLookupFallback(struct LxmlFallbackElementClassLookup *, struct LxmlDocument *, xmlNode *); +__PYX_EXTERN_C int tagMatches(xmlNode *, const xmlChar *, const xmlChar *); +__PYX_EXTERN_C struct LxmlDocument *documentOrRaise(PyObject *); +__PYX_EXTERN_C struct LxmlElement *rootNodeOrRaise(PyObject *); +__PYX_EXTERN_C int hasText(xmlNode *); +__PYX_EXTERN_C int hasTail(xmlNode *); +__PYX_EXTERN_C PyObject *textOf(xmlNode *); +__PYX_EXTERN_C PyObject *tailOf(xmlNode *); +__PYX_EXTERN_C int setNodeText(xmlNode *, PyObject *); +__PYX_EXTERN_C int setTailText(xmlNode *, PyObject *); +__PYX_EXTERN_C PyObject *attributeValue(xmlNode *, xmlAttr *); +__PYX_EXTERN_C PyObject *attributeValueFromNsName(xmlNode *, const xmlChar *, const xmlChar *); +__PYX_EXTERN_C PyObject *getAttributeValue(struct LxmlElement *, PyObject *, PyObject *); +__PYX_EXTERN_C PyObject *iterattributes(struct LxmlElement *, int); +__PYX_EXTERN_C PyObject *collectAttributes(xmlNode *, int); +__PYX_EXTERN_C int setAttributeValue(struct LxmlElement *, PyObject *, PyObject *); +__PYX_EXTERN_C int delAttribute(struct LxmlElement *, PyObject *); +__PYX_EXTERN_C int delAttributeFromNsName(xmlNode *, const xmlChar *, const xmlChar *); +__PYX_EXTERN_C int hasChild(xmlNode *); +__PYX_EXTERN_C xmlNode *findChild(xmlNode *, Py_ssize_t); +__PYX_EXTERN_C xmlNode *findChildForwards(xmlNode *, Py_ssize_t); +__PYX_EXTERN_C xmlNode *findChildBackwards(xmlNode *, Py_ssize_t); +__PYX_EXTERN_C xmlNode *nextElement(xmlNode *); +__PYX_EXTERN_C xmlNode *previousElement(xmlNode *); +__PYX_EXTERN_C void appendChild(struct LxmlElement *, struct LxmlElement *); +__PYX_EXTERN_C int appendChildToElement(struct LxmlElement *, struct LxmlElement *); +__PYX_EXTERN_C PyObject *pyunicode(const xmlChar *); +__PYX_EXTERN_C PyObject *utf8(PyObject *); +__PYX_EXTERN_C PyObject *getNsTag(PyObject *); +__PYX_EXTERN_C PyObject *getNsTagWithEmptyNs(PyObject *); +__PYX_EXTERN_C PyObject *namespacedName(xmlNode *); +__PYX_EXTERN_C PyObject *namespacedNameFromNsName(const xmlChar *, const xmlChar *); +__PYX_EXTERN_C void iteratorStoreNext(struct LxmlElementIterator *, struct LxmlElement *); +__PYX_EXTERN_C void initTagMatch(struct LxmlElementTagMatcher *, PyObject *); +__PYX_EXTERN_C xmlNs *findOrBuildNodeNsPrefix(struct LxmlDocument *, xmlNode *, const xmlChar *, const xmlChar *); + +#endif /* !__PYX_HAVE_API__lxml__etree */ + +/* WARNING: the interface of the module init function changed in CPython 3.5. */ +/* It now returns a PyModuleDef instance instead of a PyModule instance. */ + +#if PY_MAJOR_VERSION < 3 +PyMODINIT_FUNC initetree(void); +#else +PyMODINIT_FUNC PyInit_etree(void); +#endif + +#endif /* !__PYX_HAVE__lxml__etree */ diff --git a/WebCrawler/venv/Lib/site-packages/lxml/etree_api.h b/WebCrawler/venv/Lib/site-packages/lxml/etree_api.h new file mode 100644 index 0000000..729d217 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/lxml/etree_api.h @@ -0,0 +1,219 @@ +/* Generated by Cython 0.29.21 */ + +#ifndef __PYX_HAVE_API__lxml__etree +#define __PYX_HAVE_API__lxml__etree +#ifdef __MINGW64__ +#define MS_WIN64 +#endif +#include "Python.h" +#include "etree.h" + +static struct LxmlElement *(*__pyx_api_f_4lxml_5etree_deepcopyNodeToDocument)(struct LxmlDocument *, xmlNode *) = 0; +#define deepcopyNodeToDocument __pyx_api_f_4lxml_5etree_deepcopyNodeToDocument +static struct LxmlElementTree *(*__pyx_api_f_4lxml_5etree_elementTreeFactory)(struct LxmlElement *) = 0; +#define elementTreeFactory __pyx_api_f_4lxml_5etree_elementTreeFactory +static struct LxmlElementTree *(*__pyx_api_f_4lxml_5etree_newElementTree)(struct LxmlElement *, PyObject *) = 0; +#define newElementTree __pyx_api_f_4lxml_5etree_newElementTree +static struct LxmlElementTree *(*__pyx_api_f_4lxml_5etree_adoptExternalDocument)(xmlDoc *, PyObject *, int) = 0; +#define adoptExternalDocument __pyx_api_f_4lxml_5etree_adoptExternalDocument +static struct LxmlElement *(*__pyx_api_f_4lxml_5etree_elementFactory)(struct LxmlDocument *, xmlNode *) = 0; +#define elementFactory __pyx_api_f_4lxml_5etree_elementFactory +static struct LxmlElement *(*__pyx_api_f_4lxml_5etree_makeElement)(PyObject *, struct LxmlDocument *, PyObject *, PyObject *, PyObject *, PyObject *, PyObject *) = 0; +#define makeElement __pyx_api_f_4lxml_5etree_makeElement +static struct LxmlElement *(*__pyx_api_f_4lxml_5etree_makeSubElement)(struct LxmlElement *, PyObject *, PyObject *, PyObject *, PyObject *, PyObject *) = 0; +#define makeSubElement __pyx_api_f_4lxml_5etree_makeSubElement +static void (*__pyx_api_f_4lxml_5etree_setElementClassLookupFunction)(_element_class_lookup_function, PyObject *) = 0; +#define setElementClassLookupFunction __pyx_api_f_4lxml_5etree_setElementClassLookupFunction +static PyObject *(*__pyx_api_f_4lxml_5etree_lookupDefaultElementClass)(PyObject *, PyObject *, xmlNode *) = 0; +#define lookupDefaultElementClass __pyx_api_f_4lxml_5etree_lookupDefaultElementClass +static PyObject *(*__pyx_api_f_4lxml_5etree_lookupNamespaceElementClass)(PyObject *, PyObject *, xmlNode *) = 0; +#define lookupNamespaceElementClass __pyx_api_f_4lxml_5etree_lookupNamespaceElementClass +static PyObject *(*__pyx_api_f_4lxml_5etree_callLookupFallback)(struct LxmlFallbackElementClassLookup *, struct LxmlDocument *, xmlNode *) = 0; +#define callLookupFallback __pyx_api_f_4lxml_5etree_callLookupFallback +static int (*__pyx_api_f_4lxml_5etree_tagMatches)(xmlNode *, const xmlChar *, const xmlChar *) = 0; +#define tagMatches __pyx_api_f_4lxml_5etree_tagMatches +static struct LxmlDocument *(*__pyx_api_f_4lxml_5etree_documentOrRaise)(PyObject *) = 0; +#define documentOrRaise __pyx_api_f_4lxml_5etree_documentOrRaise +static struct LxmlElement *(*__pyx_api_f_4lxml_5etree_rootNodeOrRaise)(PyObject *) = 0; +#define rootNodeOrRaise __pyx_api_f_4lxml_5etree_rootNodeOrRaise +static int (*__pyx_api_f_4lxml_5etree_hasText)(xmlNode *) = 0; +#define hasText __pyx_api_f_4lxml_5etree_hasText +static int (*__pyx_api_f_4lxml_5etree_hasTail)(xmlNode *) = 0; +#define hasTail __pyx_api_f_4lxml_5etree_hasTail +static PyObject *(*__pyx_api_f_4lxml_5etree_textOf)(xmlNode *) = 0; +#define textOf __pyx_api_f_4lxml_5etree_textOf +static PyObject *(*__pyx_api_f_4lxml_5etree_tailOf)(xmlNode *) = 0; +#define tailOf __pyx_api_f_4lxml_5etree_tailOf +static int (*__pyx_api_f_4lxml_5etree_setNodeText)(xmlNode *, PyObject *) = 0; +#define setNodeText __pyx_api_f_4lxml_5etree_setNodeText +static int (*__pyx_api_f_4lxml_5etree_setTailText)(xmlNode *, PyObject *) = 0; +#define setTailText __pyx_api_f_4lxml_5etree_setTailText +static PyObject *(*__pyx_api_f_4lxml_5etree_attributeValue)(xmlNode *, xmlAttr *) = 0; +#define attributeValue __pyx_api_f_4lxml_5etree_attributeValue +static PyObject *(*__pyx_api_f_4lxml_5etree_attributeValueFromNsName)(xmlNode *, const xmlChar *, const xmlChar *) = 0; +#define attributeValueFromNsName __pyx_api_f_4lxml_5etree_attributeValueFromNsName +static PyObject *(*__pyx_api_f_4lxml_5etree_getAttributeValue)(struct LxmlElement *, PyObject *, PyObject *) = 0; +#define getAttributeValue __pyx_api_f_4lxml_5etree_getAttributeValue +static PyObject *(*__pyx_api_f_4lxml_5etree_iterattributes)(struct LxmlElement *, int) = 0; +#define iterattributes __pyx_api_f_4lxml_5etree_iterattributes +static PyObject *(*__pyx_api_f_4lxml_5etree_collectAttributes)(xmlNode *, int) = 0; +#define collectAttributes __pyx_api_f_4lxml_5etree_collectAttributes +static int (*__pyx_api_f_4lxml_5etree_setAttributeValue)(struct LxmlElement *, PyObject *, PyObject *) = 0; +#define setAttributeValue __pyx_api_f_4lxml_5etree_setAttributeValue +static int (*__pyx_api_f_4lxml_5etree_delAttribute)(struct LxmlElement *, PyObject *) = 0; +#define delAttribute __pyx_api_f_4lxml_5etree_delAttribute +static int (*__pyx_api_f_4lxml_5etree_delAttributeFromNsName)(xmlNode *, const xmlChar *, const xmlChar *) = 0; +#define delAttributeFromNsName __pyx_api_f_4lxml_5etree_delAttributeFromNsName +static int (*__pyx_api_f_4lxml_5etree_hasChild)(xmlNode *) = 0; +#define hasChild __pyx_api_f_4lxml_5etree_hasChild +static xmlNode *(*__pyx_api_f_4lxml_5etree_findChild)(xmlNode *, Py_ssize_t) = 0; +#define findChild __pyx_api_f_4lxml_5etree_findChild +static xmlNode *(*__pyx_api_f_4lxml_5etree_findChildForwards)(xmlNode *, Py_ssize_t) = 0; +#define findChildForwards __pyx_api_f_4lxml_5etree_findChildForwards +static xmlNode *(*__pyx_api_f_4lxml_5etree_findChildBackwards)(xmlNode *, Py_ssize_t) = 0; +#define findChildBackwards __pyx_api_f_4lxml_5etree_findChildBackwards +static xmlNode *(*__pyx_api_f_4lxml_5etree_nextElement)(xmlNode *) = 0; +#define nextElement __pyx_api_f_4lxml_5etree_nextElement +static xmlNode *(*__pyx_api_f_4lxml_5etree_previousElement)(xmlNode *) = 0; +#define previousElement __pyx_api_f_4lxml_5etree_previousElement +static void (*__pyx_api_f_4lxml_5etree_appendChild)(struct LxmlElement *, struct LxmlElement *) = 0; +#define appendChild __pyx_api_f_4lxml_5etree_appendChild +static int (*__pyx_api_f_4lxml_5etree_appendChildToElement)(struct LxmlElement *, struct LxmlElement *) = 0; +#define appendChildToElement __pyx_api_f_4lxml_5etree_appendChildToElement +static PyObject *(*__pyx_api_f_4lxml_5etree_pyunicode)(const xmlChar *) = 0; +#define pyunicode __pyx_api_f_4lxml_5etree_pyunicode +static PyObject *(*__pyx_api_f_4lxml_5etree_utf8)(PyObject *) = 0; +#define utf8 __pyx_api_f_4lxml_5etree_utf8 +static PyObject *(*__pyx_api_f_4lxml_5etree_getNsTag)(PyObject *) = 0; +#define getNsTag __pyx_api_f_4lxml_5etree_getNsTag +static PyObject *(*__pyx_api_f_4lxml_5etree_getNsTagWithEmptyNs)(PyObject *) = 0; +#define getNsTagWithEmptyNs __pyx_api_f_4lxml_5etree_getNsTagWithEmptyNs +static PyObject *(*__pyx_api_f_4lxml_5etree_namespacedName)(xmlNode *) = 0; +#define namespacedName __pyx_api_f_4lxml_5etree_namespacedName +static PyObject *(*__pyx_api_f_4lxml_5etree_namespacedNameFromNsName)(const xmlChar *, const xmlChar *) = 0; +#define namespacedNameFromNsName __pyx_api_f_4lxml_5etree_namespacedNameFromNsName +static void (*__pyx_api_f_4lxml_5etree_iteratorStoreNext)(struct LxmlElementIterator *, struct LxmlElement *) = 0; +#define iteratorStoreNext __pyx_api_f_4lxml_5etree_iteratorStoreNext +static void (*__pyx_api_f_4lxml_5etree_initTagMatch)(struct LxmlElementTagMatcher *, PyObject *) = 0; +#define initTagMatch __pyx_api_f_4lxml_5etree_initTagMatch +static xmlNs *(*__pyx_api_f_4lxml_5etree_findOrBuildNodeNsPrefix)(struct LxmlDocument *, xmlNode *, const xmlChar *, const xmlChar *) = 0; +#define findOrBuildNodeNsPrefix __pyx_api_f_4lxml_5etree_findOrBuildNodeNsPrefix +#if !defined(__Pyx_PyIdentifier_FromString) +#if PY_MAJOR_VERSION < 3 + #define __Pyx_PyIdentifier_FromString(s) PyString_FromString(s) +#else + #define __Pyx_PyIdentifier_FromString(s) PyUnicode_FromString(s) +#endif +#endif + +#ifndef __PYX_HAVE_RT_ImportFunction +#define __PYX_HAVE_RT_ImportFunction +static int __Pyx_ImportFunction(PyObject *module, const char *funcname, void (**f)(void), const char *sig) { + PyObject *d = 0; + PyObject *cobj = 0; + union { + void (*fp)(void); + void *p; + } tmp; + d = PyObject_GetAttrString(module, (char *)"__pyx_capi__"); + if (!d) + goto bad; + cobj = PyDict_GetItemString(d, funcname); + if (!cobj) { + PyErr_Format(PyExc_ImportError, + "%.200s does not export expected C function %.200s", + PyModule_GetName(module), funcname); + goto bad; + } +#if PY_VERSION_HEX >= 0x02070000 + if (!PyCapsule_IsValid(cobj, sig)) { + PyErr_Format(PyExc_TypeError, + "C function %.200s.%.200s has wrong signature (expected %.500s, got %.500s)", + PyModule_GetName(module), funcname, sig, PyCapsule_GetName(cobj)); + goto bad; + } + tmp.p = PyCapsule_GetPointer(cobj, sig); +#else + {const char *desc, *s1, *s2; + desc = (const char *)PyCObject_GetDesc(cobj); + if (!desc) + goto bad; + s1 = desc; s2 = sig; + while (*s1 != '\0' && *s1 == *s2) { s1++; s2++; } + if (*s1 != *s2) { + PyErr_Format(PyExc_TypeError, + "C function %.200s.%.200s has wrong signature (expected %.500s, got %.500s)", + PyModule_GetName(module), funcname, sig, desc); + goto bad; + } + tmp.p = PyCObject_AsVoidPtr(cobj);} +#endif + *f = tmp.fp; + if (!(*f)) + goto bad; + Py_DECREF(d); + return 0; +bad: + Py_XDECREF(d); + return -1; +} +#endif + + +static int import_lxml__etree(void) { + PyObject *module = 0; + module = PyImport_ImportModule("lxml.etree"); + if (!module) goto bad; + if (__Pyx_ImportFunction(module, "deepcopyNodeToDocument", (void (**)(void))&__pyx_api_f_4lxml_5etree_deepcopyNodeToDocument, "struct LxmlElement *(struct LxmlDocument *, xmlNode *)") < 0) goto bad; + if (__Pyx_ImportFunction(module, "elementTreeFactory", (void (**)(void))&__pyx_api_f_4lxml_5etree_elementTreeFactory, "struct LxmlElementTree *(struct LxmlElement *)") < 0) goto bad; + if (__Pyx_ImportFunction(module, "newElementTree", (void (**)(void))&__pyx_api_f_4lxml_5etree_newElementTree, "struct LxmlElementTree *(struct LxmlElement *, PyObject *)") < 0) goto bad; + if (__Pyx_ImportFunction(module, "adoptExternalDocument", (void (**)(void))&__pyx_api_f_4lxml_5etree_adoptExternalDocument, "struct LxmlElementTree *(xmlDoc *, PyObject *, int)") < 0) goto bad; + if (__Pyx_ImportFunction(module, "elementFactory", (void (**)(void))&__pyx_api_f_4lxml_5etree_elementFactory, "struct LxmlElement *(struct LxmlDocument *, xmlNode *)") < 0) goto bad; + if (__Pyx_ImportFunction(module, "makeElement", (void (**)(void))&__pyx_api_f_4lxml_5etree_makeElement, "struct LxmlElement *(PyObject *, struct LxmlDocument *, PyObject *, PyObject *, PyObject *, PyObject *, PyObject *)") < 0) goto bad; + if (__Pyx_ImportFunction(module, "makeSubElement", (void (**)(void))&__pyx_api_f_4lxml_5etree_makeSubElement, "struct LxmlElement *(struct LxmlElement *, PyObject *, PyObject *, PyObject *, PyObject *, PyObject *)") < 0) goto bad; + if (__Pyx_ImportFunction(module, "setElementClassLookupFunction", (void (**)(void))&__pyx_api_f_4lxml_5etree_setElementClassLookupFunction, "void (_element_class_lookup_function, PyObject *)") < 0) goto bad; + if (__Pyx_ImportFunction(module, "lookupDefaultElementClass", (void (**)(void))&__pyx_api_f_4lxml_5etree_lookupDefaultElementClass, "PyObject *(PyObject *, PyObject *, xmlNode *)") < 0) goto bad; + if (__Pyx_ImportFunction(module, "lookupNamespaceElementClass", (void (**)(void))&__pyx_api_f_4lxml_5etree_lookupNamespaceElementClass, "PyObject *(PyObject *, PyObject *, xmlNode *)") < 0) goto bad; + if (__Pyx_ImportFunction(module, "callLookupFallback", (void (**)(void))&__pyx_api_f_4lxml_5etree_callLookupFallback, "PyObject *(struct LxmlFallbackElementClassLookup *, struct LxmlDocument *, xmlNode *)") < 0) goto bad; + if (__Pyx_ImportFunction(module, "tagMatches", (void (**)(void))&__pyx_api_f_4lxml_5etree_tagMatches, "int (xmlNode *, const xmlChar *, const xmlChar *)") < 0) goto bad; + if (__Pyx_ImportFunction(module, "documentOrRaise", (void (**)(void))&__pyx_api_f_4lxml_5etree_documentOrRaise, "struct LxmlDocument *(PyObject *)") < 0) goto bad; + if (__Pyx_ImportFunction(module, "rootNodeOrRaise", (void (**)(void))&__pyx_api_f_4lxml_5etree_rootNodeOrRaise, "struct LxmlElement *(PyObject *)") < 0) goto bad; + if (__Pyx_ImportFunction(module, "hasText", (void (**)(void))&__pyx_api_f_4lxml_5etree_hasText, "int (xmlNode *)") < 0) goto bad; + if (__Pyx_ImportFunction(module, "hasTail", (void (**)(void))&__pyx_api_f_4lxml_5etree_hasTail, "int (xmlNode *)") < 0) goto bad; + if (__Pyx_ImportFunction(module, "textOf", (void (**)(void))&__pyx_api_f_4lxml_5etree_textOf, "PyObject *(xmlNode *)") < 0) goto bad; + if (__Pyx_ImportFunction(module, "tailOf", (void (**)(void))&__pyx_api_f_4lxml_5etree_tailOf, "PyObject *(xmlNode *)") < 0) goto bad; + if (__Pyx_ImportFunction(module, "setNodeText", (void (**)(void))&__pyx_api_f_4lxml_5etree_setNodeText, "int (xmlNode *, PyObject *)") < 0) goto bad; + if (__Pyx_ImportFunction(module, "setTailText", (void (**)(void))&__pyx_api_f_4lxml_5etree_setTailText, "int (xmlNode *, PyObject *)") < 0) goto bad; + if (__Pyx_ImportFunction(module, "attributeValue", (void (**)(void))&__pyx_api_f_4lxml_5etree_attributeValue, "PyObject *(xmlNode *, xmlAttr *)") < 0) goto bad; + if (__Pyx_ImportFunction(module, "attributeValueFromNsName", (void (**)(void))&__pyx_api_f_4lxml_5etree_attributeValueFromNsName, "PyObject *(xmlNode *, const xmlChar *, const xmlChar *)") < 0) goto bad; + if (__Pyx_ImportFunction(module, "getAttributeValue", (void (**)(void))&__pyx_api_f_4lxml_5etree_getAttributeValue, "PyObject *(struct LxmlElement *, PyObject *, PyObject *)") < 0) goto bad; + if (__Pyx_ImportFunction(module, "iterattributes", (void (**)(void))&__pyx_api_f_4lxml_5etree_iterattributes, "PyObject *(struct LxmlElement *, int)") < 0) goto bad; + if (__Pyx_ImportFunction(module, "collectAttributes", (void (**)(void))&__pyx_api_f_4lxml_5etree_collectAttributes, "PyObject *(xmlNode *, int)") < 0) goto bad; + if (__Pyx_ImportFunction(module, "setAttributeValue", (void (**)(void))&__pyx_api_f_4lxml_5etree_setAttributeValue, "int (struct LxmlElement *, PyObject *, PyObject *)") < 0) goto bad; + if (__Pyx_ImportFunction(module, "delAttribute", (void (**)(void))&__pyx_api_f_4lxml_5etree_delAttribute, "int (struct LxmlElement *, PyObject *)") < 0) goto bad; + if (__Pyx_ImportFunction(module, "delAttributeFromNsName", (void (**)(void))&__pyx_api_f_4lxml_5etree_delAttributeFromNsName, "int (xmlNode *, const xmlChar *, const xmlChar *)") < 0) goto bad; + if (__Pyx_ImportFunction(module, "hasChild", (void (**)(void))&__pyx_api_f_4lxml_5etree_hasChild, "int (xmlNode *)") < 0) goto bad; + if (__Pyx_ImportFunction(module, "findChild", (void (**)(void))&__pyx_api_f_4lxml_5etree_findChild, "xmlNode *(xmlNode *, Py_ssize_t)") < 0) goto bad; + if (__Pyx_ImportFunction(module, "findChildForwards", (void (**)(void))&__pyx_api_f_4lxml_5etree_findChildForwards, "xmlNode *(xmlNode *, Py_ssize_t)") < 0) goto bad; + if (__Pyx_ImportFunction(module, "findChildBackwards", (void (**)(void))&__pyx_api_f_4lxml_5etree_findChildBackwards, "xmlNode *(xmlNode *, Py_ssize_t)") < 0) goto bad; + if (__Pyx_ImportFunction(module, "nextElement", (void (**)(void))&__pyx_api_f_4lxml_5etree_nextElement, "xmlNode *(xmlNode *)") < 0) goto bad; + if (__Pyx_ImportFunction(module, "previousElement", (void (**)(void))&__pyx_api_f_4lxml_5etree_previousElement, "xmlNode *(xmlNode *)") < 0) goto bad; + if (__Pyx_ImportFunction(module, "appendChild", (void (**)(void))&__pyx_api_f_4lxml_5etree_appendChild, "void (struct LxmlElement *, struct LxmlElement *)") < 0) goto bad; + if (__Pyx_ImportFunction(module, "appendChildToElement", (void (**)(void))&__pyx_api_f_4lxml_5etree_appendChildToElement, "int (struct LxmlElement *, struct LxmlElement *)") < 0) goto bad; + if (__Pyx_ImportFunction(module, "pyunicode", (void (**)(void))&__pyx_api_f_4lxml_5etree_pyunicode, "PyObject *(const xmlChar *)") < 0) goto bad; + if (__Pyx_ImportFunction(module, "utf8", (void (**)(void))&__pyx_api_f_4lxml_5etree_utf8, "PyObject *(PyObject *)") < 0) goto bad; + if (__Pyx_ImportFunction(module, "getNsTag", (void (**)(void))&__pyx_api_f_4lxml_5etree_getNsTag, "PyObject *(PyObject *)") < 0) goto bad; + if (__Pyx_ImportFunction(module, "getNsTagWithEmptyNs", (void (**)(void))&__pyx_api_f_4lxml_5etree_getNsTagWithEmptyNs, "PyObject *(PyObject *)") < 0) goto bad; + if (__Pyx_ImportFunction(module, "namespacedName", (void (**)(void))&__pyx_api_f_4lxml_5etree_namespacedName, "PyObject *(xmlNode *)") < 0) goto bad; + if (__Pyx_ImportFunction(module, "namespacedNameFromNsName", (void (**)(void))&__pyx_api_f_4lxml_5etree_namespacedNameFromNsName, "PyObject *(const xmlChar *, const xmlChar *)") < 0) goto bad; + if (__Pyx_ImportFunction(module, "iteratorStoreNext", (void (**)(void))&__pyx_api_f_4lxml_5etree_iteratorStoreNext, "void (struct LxmlElementIterator *, struct LxmlElement *)") < 0) goto bad; + if (__Pyx_ImportFunction(module, "initTagMatch", (void (**)(void))&__pyx_api_f_4lxml_5etree_initTagMatch, "void (struct LxmlElementTagMatcher *, PyObject *)") < 0) goto bad; + if (__Pyx_ImportFunction(module, "findOrBuildNodeNsPrefix", (void (**)(void))&__pyx_api_f_4lxml_5etree_findOrBuildNodeNsPrefix, "xmlNs *(struct LxmlDocument *, xmlNode *, const xmlChar *, const xmlChar *)") < 0) goto bad; + Py_DECREF(module); module = 0; + return 0; + bad: + Py_XDECREF(module); + return -1; +} + +#endif /* !__PYX_HAVE_API__lxml__etree */ diff --git a/WebCrawler/venv/Lib/site-packages/lxml/html/ElementSoup.py b/WebCrawler/venv/Lib/site-packages/lxml/html/ElementSoup.py new file mode 100644 index 0000000..c35365d --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/lxml/html/ElementSoup.py @@ -0,0 +1,10 @@ +__doc__ = """Legacy interface to the BeautifulSoup HTML parser. +""" + +__all__ = ["parse", "convert_tree"] + +from .soupparser import convert_tree, parse as _parse + +def parse(file, beautifulsoup=None, makeelement=None): + root = _parse(file, beautifulsoup=beautifulsoup, makeelement=makeelement) + return root.getroot() diff --git a/WebCrawler/venv/Lib/site-packages/lxml/html/__init__.py b/WebCrawler/venv/Lib/site-packages/lxml/html/__init__.py new file mode 100644 index 0000000..2139c75 --- /dev/null +++ b/WebCrawler/venv/Lib/site-packages/lxml/html/__init__.py @@ -0,0 +1,1948 @@ +# Copyright (c) 2004 Ian Bicking. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# +# 3. Neither the name of Ian Bicking nor the names of its contributors may +# be used to endorse or promote products derived from this software +# without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL IAN BICKING OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""The ``lxml.html`` tool set for HTML handling. +""" + +from __future__ import absolute_import + +__all__ = [ + 'document_fromstring', 'fragment_fromstring', 'fragments_fromstring', 'fromstring', + 'tostring', 'Element', 'defs', 'open_in_browser', 'submit_form', + 'find_rel_links', 'find_class', 'make_links_absolute', + 'resolve_base_href', 'iterlinks', 'rewrite_links', 'parse'] + + +import copy +import sys +import re +from functools import partial + +try: + from collections.abc import MutableMapping, MutableSet +except ImportError: + from collections import MutableMapping, MutableSet + +from .. import etree +from . import defs +from ._setmixin import SetMixin + +try: + from urlparse import urljoin +except ImportError: + # Python 3 + from urllib.parse import urljoin + +try: + unicode +except NameError: + # Python 3 + unicode = str +try: + basestring +except NameError: + # Python 3 + basestring = (str, bytes) + + +def __fix_docstring(s): + if not s: + return s + if sys.version_info[0] >= 3: + sub = re.compile(r"^(\s*)u'", re.M).sub + else: + sub = re.compile(r"^(\s*)b'", re.M).sub + return sub(r"\1'", s) + + +XHTML_NAMESPACE = "http://www.w3.org/1999/xhtml" + +_rel_links_xpath = etree.XPath("descendant-or-self::a[@rel]|descendant-or-self::x:a[@rel]", + namespaces={'x':XHTML_NAMESPACE}) +_options_xpath = etree.XPath("descendant-or-self::option|descendant-or-self::x:option", + namespaces={'x':XHTML_NAMESPACE}) +_forms_xpath = etree.XPath("descendant-or-self::form|descendant-or-self::x:form", + namespaces={'x':XHTML_NAMESPACE}) +#_class_xpath = etree.XPath(r"descendant-or-self::*[regexp:match(@class, concat('\b', $class_name, '\b'))]", {'regexp': 'http://exslt.org/regular-expressions'}) +_class_xpath = etree.XPath("descendant-or-self::*[@class and contains(concat(' ', normalize-space(@class), ' '), concat(' ', $class_name, ' '))]") +_id_xpath = etree.XPath("descendant-or-self::*[@id=$id]") +_collect_string_content = etree.XPath("string()") +_iter_css_urls = re.compile(r'url\(('+'["][^"]*["]|'+"['][^']*[']|"+r'[^)]*)\)', re.I).finditer +_iter_css_imports = re.compile(r'@import "(.*?)"').finditer +_label_xpath = etree.XPath("//label[@for=$id]|//x:label[@for=$id]", + namespaces={'x':XHTML_NAMESPACE}) +_archive_re = re.compile(r'[^ ]+') +_parse_meta_refresh_url = re.compile( + r'[^;=]*;\s*(?:url\s*=\s*)?(?P.*)$', re.I).search + + +def _unquote_match(s, pos): + if s[:1] == '"' and s[-1:] == '"' or s[:1] == "'" and s[-1:] == "'": + return s[1:-1], pos+1 + else: + return s,pos + + +def _transform_result(typ, result): + """Convert the result back into the input type. + """ + if issubclass(typ, bytes): + return tostring(result, encoding='utf-8') + elif issubclass(typ, unicode): + return tostring(result, encoding='unicode') + else: + return result + + +def _nons(tag): + if isinstance(tag, basestring): + if tag[0] == '{' and tag[1:len(XHTML_NAMESPACE)+1] == XHTML_NAMESPACE: + return tag.split('}')[-1] + return tag + + +class Classes(MutableSet): + """Provides access to an element's class attribute as a set-like collection. + Usage:: + + >>> el = fromstring('') + >>> classes = el.classes # or: classes = Classes(el.attrib) + >>> classes |= ['block', 'paragraph'] + >>> el.get('class') + 'hidden large block paragraph' + >>> classes.toggle('hidden') + False + >>> el.get('class') + 'large block paragraph' + >>> classes -= ('some', 'classes', 'block') + >>> el.get('class') + 'large paragraph' + """ + def __init__(self, attributes): + self._attributes = attributes + self._get_class_value = partial(attributes.get, 'class', '') + + def add(self, value): + """ + Add a class. + + This has no effect if the class is already present. + """ + if not value or re.search(r'\s', value): + raise ValueError("Invalid class name: %r" % value) + classes = self._get_class_value().split() + if value in classes: + return + classes.append(value) + self._attributes['class'] = ' '.join(classes) + + def discard(self, value): + """ + Remove a class if it is currently present. + + If the class is not present, do nothing. + """ + if not value or re.search(r'\s', value): + raise ValueError("Invalid class name: %r" % value) + classes = [name for name in self._get_class_value().split() + if name != value] + if classes: + self._attributes['class'] = ' '.join(classes) + elif 'class' in self._attributes: + del self._attributes['class'] + + def remove(self, value): + """ + Remove a class; it must currently be present. + + If the class is not present, raise a KeyError. + """ + if not value or re.search(r'\s', value): + raise ValueError("Invalid class name: %r" % value) + super(Classes, self).remove(value) + + def __contains__(self, name): + classes = self._get_class_value() + return name in classes and name in classes.split() + + def __iter__(self): + return iter(self._get_class_value().split()) + + def __len__(self): + return len(self._get_class_value().split()) + + # non-standard methods + + def update(self, values): + """ + Add all names from 'values'. + """ + classes = self._get_class_value().split() + extended = False + for value in values: + if value not in classes: + classes.append(value) + extended = True + if extended: + self._attributes['class'] = ' '.join(classes) + + def toggle(self, value): + """ + Add a class name if it isn't there yet, or remove it if it exists. + + Returns true if the class was added (and is now enabled) and + false if it was removed (and is now disabled). + """ + if not value or re.search(r'\s', value): + raise ValueError("Invalid class name: %r" % value) + classes = self._get_class_value().split() + try: + classes.remove(value) + enabled = False + except ValueError: + classes.append(value) + enabled = True + if classes: + self._attributes['class'] = ' '.join(classes) + else: + del self._attributes['class'] + return enabled + + +class HtmlMixin(object): + + def set(self, key, value=None): + """set(self, key, value=None) + + Sets an element attribute. If no value is provided, or if the value is None, + creates a 'boolean' attribute without value, e.g. "
" + for ``form.set('novalidate')``. + """ + super(HtmlElement, self).set(key, value) + + @property + def classes(self): + """ + A set-like wrapper around the 'class' attribute. + """ + return Classes(self.attrib) + + @classes.setter + def classes(self, classes): + assert isinstance(classes, Classes) # only allow "el.classes |= ..." etc. + value = classes._get_class_value() + if value: + self.set('class', value) + elif self.get('class') is not None: + del self.attrib['class'] + + @property + def base_url(self): + """ + Returns the base URL, given when the page was parsed. + + Use with ``urlparse.urljoin(el.base_url, href)`` to get + absolute URLs. + """ + return self.getroottree().docinfo.URL + + @property + def forms(self): + """ + Return a list of all the forms + """ + return _forms_xpath(self) + + @property + def body(self): + """ + Return the element. Can be called from a child element + to get the document's head. + """ + return self.xpath('//body|//x:body', namespaces={'x':XHTML_NAMESPACE})[0] + + @property + def head(self): + """ + Returns the element. Can be called from a child + element to get the document's head. + """ + return self.xpath('//head|//x:head', namespaces={'x':XHTML_NAMESPACE})[0] + + @property + def label(self): + """ + Get or set any