[API/web scraping project]Max Guzmán

your-code/output/API_ssd.py

@@ -0,0 +1,28 @@
+import requests
+import pandas as pd
+
+ruta_archivo = r'..\output\objetos_cercanos.csv'
+
+url = 'https://ssd-api.jpl.nasa.gov/cad.api?body=all'
+res = requests.get(url)
+res_json = res.json()
+
+res_json.keys()
+
+fields = res_json['fields']
+data = res_json['data']
+
+df = pd.DataFrame(data, columns=fields)
+
+df = df[['des','orbit_id','cd','dist','v_rel','t_sigma_f','body','h']]
+
+columnas = ['Designacion', 'id_orbita', 'Fecha_aproximacion_cercana', 
+'Distancia_au', 'Velocidad_relativa_km/s','Incertidumbre_temporal',
+'Cuerpo','Mgnitud_absoluta']
+
+df.columns = columnas
+
+print(df.info())
+print(df.head())
+
+df.to_csv(ruta_archivo,index=False)

your-code/output/articulos_electronica.csv

@@ -0,0 +1,49 @@
+articulo,precio
+40 Cables Dupont Macho a Macho 20 cm,$ 55.00
+40 Cables Dupont Macho a Hembra 20 cm,$ 40.00
+Arduino Uno R3 ATmega328 Compatible,$ 300.00
+Arduino nano ATmega328 Compatible,$ 140.00
+40 Cables Dupont Hembra a Hembra 20 cm,$ 35.00
+40 Cables Dupont Macho a Macho de 10 cm,$ 30.00
+40 Cables Dupont Macho a Hembra de 10 CM,$ 30.00
+Arduino Mega ATmega2560 compatible,$ 580.00
+40 Cables Dupont Macho a Hembra de 30 cm,$ 50.00
+Convertidor USB A TTL PL2303,$ 35.00
+40 Cables Dupont Macho a Macho 30 cm,$ 50.00
+40 Cables Dupont Hembra a Hembra de 10 cm,$ 30.00
+65 Cables Dupont Macho a Macho,$ 45.00
+Fuente regulada LM2596,$ 40.00
+Ethernet Shield con conector micro SD,$ 520.00
+Kit de arduino básico 263 elementos,"$ 1,400.00"
+Arduino Pro Mini 328 - 5V/16MHz,$ 120.00
+40 Cables Dupont Hembra a Hembra 30 cm,$ 50.00
+Carcasa de acrílico para Arduino Uno Talos,$ 80.00
+Arduino Shield L293D,$ 90.00
+Fuente de voltaje ajustable DC-DC 5A,$ 120.00
+Arduino UNO Sensor Shield Edición V5,$ 60.00
+Arduino Leonardo ATmega32u4 Compatible,$ 260.00
+Programador USB ISP 51/AVR,$ 85.00
+Kit sensores Arduino 37pcs,$ 800.00
+Módulo Ethernet ENC28J60 para Arduino,$ 160.00
+CNC Shield V3 para Arduino,$ 95.00
+Kit de Arduino RFID 159 elementos,"$ 1,249.00"
+Protoshield para arduino UNO,$ 70.00
+Tornillo Shield V1 para Arduino UNO y Leonardo,$ 150.00
+Carcasa de acrílico para Arduino MEGA Talos,$ 90.00
+LCD y Teclado Shield para Arduino,$ 145.00
+Arduino MEGA Sensor Shield V2.0,$ 110.00
+Módulo Bluetooth XBee FT232RL + Cable USB,$ 195.00
+Game Joystick Arduino Shield,$ 160.00
+Tornillo Shield V2 para Arduino UNO y Leonardo,$ 140.00
+Arduino DUE ATSAM3X8E Compatible + Cable,$ 620.00
+Fuente de voltaje ajustable LM2577,$ 125.00
+Módulo CH340G USB a TTL STC,$ 45.00
+Módulo GPS Neo GPS6MV2,$ 200.00
+Base Arduino MEGA y proto 830pts,$ 115.00
+Carcasa de acrílico para Arduino UNO Tipo B Talos,$ 95.00
+Fuente de voltaje ajustable step-down MP1584,$ 30.00
+Fuente voltaje y corriente Ajustable XL4015,$ 120.00
+Fuente de voltaje ajustable 10-32V a 12-35V,$ 110.00
+Módulo Bluetooh xBee B3.0,$ 280.00
+Carcasa de acrílico para Arduino MEGA Tipo B Talos,$ 110.00
+Carcasa de Plástico ABS para Arduino Uno transparente,$ 120.00

your-code/output/objetos_cercanos.csv

@@ -0,0 +1,53 @@
+Designacion,id_orbita,Fecha_aproximacion_cercana,Distancia_au,Velocidad_relativa_km/s,Incertidumbre_temporal,Cuerpo,Mgnitud_absoluta
+2016 JG12,41,2021-Nov-20 00:23,0.0369165706489928,7.47734218550301,< 00:01,Earth,22.53
+2021 TR15,8,2021-Nov-20 00:29,0.030454045218752,6.98300523232658,< 00:01,Earth,24.63
+2021 VP10,3,2021-Nov-20 02:34,0.00462916619015841,7.88689125321877,00:07,Moon,27.869
+2021 VP10,3,2021-Nov-20 14:53,0.00514387234391849,8.2777401127482,00:07,Earth,27.869
+2021 KH2,7,2021-Nov-21 00:25,0.0494202303118363,6.51253949996388,< 00:01,Earth,25.29
+1999 XK136,29,2021-Nov-21 13:30,0.0165573454149366,12.6764985548603,00:03,Venus,20.3
+2021 VW5,4,2021-Nov-21 16:17,0.0400456526289211,3.42660310556936,00:07,Earth,26.62
+3361,172,2021-Nov-21 19:20,0.0385607791994219,8.13939947244608,< 00:01,Earth,19.24
+2021 VQ26,2,2021-Nov-22 11:58,0.0118682501245768,10.4225693967833,00:05,Earth,24.109
+2021 VL16,1,2021-Nov-23 22:28,0.0366069671769688,8.36758405318245,00:06,Earth,26.331
+2014 WF201,12,2021-Nov-24 22:45,0.0336679602889608,5.45672976550807,13:25,Earth,25.6
+2021 VR4,2,2021-Nov-25 03:25,0.00548712837324514,7.72342766651524,00:55,Earth,26.558
+2002 JE9,51,2021-Nov-25 04:30,0.00494076941567594,13.6924553471742,< 00:01,Venus,21.4
+2021 VR4,2,2021-Nov-25 06:26,0.00768975581826847,6.76357508997615,01:07,Moon,26.558
+2009 WB105,25,2021-Nov-25 06:31,0.0386856183293863,18.8886414731086,< 00:01,Earth,23.5
+2021 VF11,2,2021-Nov-25 16:05,0.0475764229872822,1.34810119116721,00:02,Earth,27.096
+2019 BB5,5,2021-Nov-25 17:18,0.0480110478137196,8.34811069174473,06:13,Earth,26.7
+2021 VM,3,2021-Nov-27 03:24,0.0239730092258416,8.93845285548012,00:15,Earth,25.38
+2020 TZ5,4,2021-Nov-27 16:14,0.0298935942626182,4.00520585755241,2_01:16,Mars,23.3
+2021 VO12,2,2021-Nov-28 15:39,0.0187515506370561,13.4915828610281,00:10,Earth,24.171
+1994 WR12,25,2021-Nov-29 06:03,0.0411219670941506,8.82373124453306,00:01,Earth,22.3
+2021 JB,6,2021-Dec-03 07:16,0.0396412654577259,6.69950605562626,09:18,Venus,26.69
+2021 UP4,2,2021-Dec-04 19:43,0.0356233448734544,8.31529986298527,00:15,Earth,24.15
+2019 JD7,13,2021-Dec-05 04:59,0.0285227461613564,8.90769885132612,00:02,Venus,21.90
+2020 XR1,5,2021-Dec-05 22:11,0.0322123051786317,29.4429833179663,11:36,Mercury,23.8
+2021 VX7,6,2021-Dec-06 02:25,0.037941077149092,6.14228152706662,00:04,Earth,24.668
+2018 YG,3,2021-Dec-06 08:11,0.0454136316438688,32.3442488879733,00:04,Mercury,24.3
+4660,181,2021-Dec-11 13:51,0.0262988224542705,6.57822864352711,< 00:01,Earth,18.35
+2019 QM5,7,2021-Dec-12 12:54,0.0243807953480248,17.9498220806853,1_00:40,Mars,21.6
+2019 XQ1,4,2021-Dec-13 13:51,0.0361183020857895,9.13438529798828,01:40,Earth,25.4
+2016 NR,12,2021-Dec-14 02:29,0.0256632719449654,11.6985791276465,00:05,Venus,23.8
+2021 VT6,4,2021-Dec-14 10:19,0.0197834720231418,6.9443963334251,02:01,Earth,24.037
+2004 YC,9,2021-Dec-15 20:01,0.0472060962949736,8.12429775498109,3_01:05,Earth,25.6
+163899,144,2021-Dec-17 17:26,0.0362814859394219,5.61061834995688,< 00:01,Earth,17.56
+2016 YY10,3,2021-Dec-21 07:31,0.028808458733509,9.24235782762831,4_11:39,Earth,25.9
+2017 XQ60,8,2021-Dec-21 08:58,0.0351743548406297,15.7427570921525,< 00:01,Earth,24.4
+2016 TR54,23,2021-Dec-24 06:30,0.0432458901998697,15.5301379579902,< 00:01,Earth,22.1
+2019 WN2,6,2021-Dec-25 02:21,0.00796383534663379,5.08168014019648,02:59,Mars,27.6
+2018 AH,12,2021-Dec-27 16:27,0.0303977239041108,12.7037995422583,3_14:40,Earth,22.5
+2017 AE3,10,2021-Dec-29 01:54,0.0236956837788724,19.0836225730656,6_06:03,Earth,21.8
+2021 BD,2,2022-Jan-01 12:53,0.0358458435867939,46.4118421763132,18:25,Mercury,26.9
+2013 GV68,34,2022-Jan-01 20:07,0.0458371118078146,28.1882731881133,00:01,Mercury,18.9
+2014 YE15,11,2022-Jan-06 03:54,0.0494909337316518,6.37967766581341,00:08,Earth,28.3
+2019 CH1,5,2022-Jan-06 05:36,0.0493138172102694,25.1981280966746,00:03,Venus,21.0
+2020 AP1,2,2022-Jan-07 17:32,0.0116565977792099,5.65179413914172,8_20:07,Earth,29.6
+2013 YD48,23,2022-Jan-11 23:39,0.0373977429962872,14.770440845013,02:25,Earth,22.6
+2008 QU3,26,2022-Jan-14 22:45,0.0347076231924351,13.3861945062755,< 00:01,Venus,20.61
+2021 BA,6,2022-Jan-18 07:54,0.0251227022635361,9.09593847332264,00:50,Earth,26.01
+480934,47,2022-Jan-18 09:31,0.0157326058413912,12.5713343835922,< 00:01,Mars,17.57
+388188,78,2022-Jan-18 12:50,0.0191587250173642,30.292528856863,< 00:01,Venus,18.9
+2015 DR215,18,2022-Jan-18 15:22,0.0396553214560677,18.7833943672152,00:01,Venus,20.44
+7482,144,2022-Jan-18 21:51,0.0132450566543752,19.5554663946525,< 00:01,Earth,16.56

your-code/output/web_scrapper_electronica.py

@@ -0,0 +1,138 @@
+from bs4 import BeautifulSoup
+import pandas as pd
+import time
+from selenium import webdriver
+from selenium.webdriver.common.keys import Keys
+from selenium.webdriver.common.action_chains import ActionChains
+import random as rd
+from selenium.webdriver.chrome.options import Options
+
+url1 = 'https://www.taloselectronics.com/collections/arduino?page=1&grid_list=grid-view'
+url2 = 'https://www.taloselectronics.com/collections/arduino?page=2&grid_list=grid-view'
+url3 = 'https://www.taloselectronics.com/collections/arduino?page=3&grid_list=grid-view'
+
+path = r'C:\Program Files\ChromeDriver\chromedriver.exe'
+ruta_archivo = r'..\output\articulos_electronica.csv'
+
+def get_source(url):
+    ''' Esta función toma una url y regresa el código fuente
+        utilizando el driver de Chrome
+    '''
+    chrome_options = Options()
+    chrome_options.add_argument("--headless")
+
+    driver = webdriver.Chrome(path, options=chrome_options)
+    driver.get(url)
+
+    rand_keys(driver)
+
+    page_source = driver.page_source
+    print('Extracción terminada')
+    driver.quit()
+
+    return page_source
+
+
+def rand_keys(driver):
+    ''' Esta función envía teclas presionadas al driver de Chrome.
+        La tecla presionada es aleatoria así como el intervalo de
+        tiempo entre cada acción.
+    '''
+    keys_list = ['END', 'HOME', 'ARROW_DOWN', 'ARROW_UP']
+    time.sleep(rd.randint(1,3))
+
+    for i in range(rd.randint(1,20)):
+        ActionChains(driver).send_keys(Keys.ARROW_DOWN).perform()
+        print('Presionando tecla')
+        time.sleep(rd.randint(1,3))
+
+# Generando la sopa
+def give_me_soup(source):
+    """ Esta función toma un listado de urls y devuelve un listado de contenidos html
+        en forma de sopa.
+    """
+    soup = BeautifulSoup(source, 'html')
+    return soup
+
+page_to_scrap = input('Ingresa el número de la página que vamos a scrapear: 1, 2 o 3')
+while page_to_scrap not in ['1','2','3']:
+    page_to_scrap = input('Ese numero no es válido. 1, 2 o 3')
+
+if page_to_scrap == '1':
+    url = url1
+elif page_to_scrap == '2':
+    url = url2
+elif page_to_scrap == '3':
+    url = url3
+
+sources = get_source(url)
+soup = give_me_soup(sources)
+
+articulo = soup.select('ul div a')
+
+# Extrayendo el link de su tag y almacenandolo en una lista
+links = [articulo[i]['href'] for i in range(len(articulo))]
+links_href = []
+for link in links:
+    if link not in links_href:
+        links_href.append(link)
+
+# Purgando la lista de links para dejar sólo artículos
+art_links = ['https://www.taloselectronics.com/'+link for link in links_href if link.startswith('/collections/arduino')]
+
+# Entrando a cada url de los artículos de la página y extrayendo su contenido
+# articles = [get_source(art_link) for art_link in art_links]
+articles = []
+for i,art_link in enumerate(art_links):
+    print(f'obteniendo el artículo num. {i} de {art_link}\n')
+    get_source(art_link)
+
+# Obteniendo el HTML de cada artículo
+art_soups = [give_me_soup(art) for art in articles]
+
+# Extrayendo nombre y precio de cada artíciulo en una lista
+articles_name = []
+articles_price = []
+for art_soup in art_soups:
+    articulo = art_soup.select('h1')
+    articulo = articulo[0].text.strip()
+    articles_name.append(articulo)
+    precio = art_soup.select('article span[class="money"]')
+    precio = precio[1].text.strip()  
+    articles_price.append(precio)
+
+# Creando un dataframe vació con dos columnas
+df = pd.DataFrame(columns=['articulo','precio'])
+
+# Creando dos tablas con nombres y precios c/u y concatenandolos en el dataframe
+articulos = pd.DataFrame(articles_name)
+precios= pd.DataFrame(articles_price)
+df = pd.concat([articulos, precios], axis=1)
+
+if page_to_scrap == '1':
+    # Guardar primera tabla
+    df.to_csv(ruta_archivo, index=False)
+    print('archivo CSV actualizado') 
+    df_anterior = df
+    print(f'Tabla de articulos en {url1}: \n')
+    df_anterior
+
+
+elif page_to_scrap == '2':
+
+    df_anterior = pd.read_csv(ruta_archivo)
+    nuevo_df = df_anterior
+    nuevo_df = pd.concat([df])
+    nuevo_df.to_csv(ruta_archivo, index=False)
+    print(f'Tabla de articulos en {url1} y {url2}: \n')
+    print('archivo CSV actualizado')  
+    nuevo_df
+
+else:
+    df_anterior = pd.read_csv(ruta_archivo)
+    nuevo_df = df_anterior
+    nuevo_df = pd.concat([df])
+    nuevo_df.to_csv(ruta_archivo, index=False)
+    print(f'Tabla de articulos en {url1}, {url2} y {url3}: \n')
+    print('archivo CSV actualizado')  
+    nuevo_df