Menguasai 3 Teknik Web Scraping: Forms, AJAX, dan iFrames

from bs4 import BeautifulSoup
import requests
import pandas as pd

BASE_URL = "https://www.scrapethissite.com/pages/forms/"
SEARCH_URL = "https://www.scrapethissite.com/pages/forms/?page={}"

def scrape_hockey_data(max_page=25):
    """Scraping seluruh data dari halaman dengan pagination."""
    data = []
    page = 1

    while page <= max_page:
        print(f"📄 Scraping halaman {page}...")
        
        response = requests.get(SEARCH_URL.format(page))
        
        if response.status_code != 200:
            print(f"❌ Gagal mengambil halaman {page}")
            break

        soup = BeautifulSoup(response.text, "html.parser")
        teams = soup.find_all("tr", class_="team")

        if not teams:
            print("✅ Tidak ada halaman berikutnya")
            break

        for team in teams:
            team_data = extract_team_data(team)
            data.append(team_data)

        page += 1

    return data

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import time

# Setup WebDriver
options = webdriver.ChromeOptions()
options.add_argument("--headless")
driver = webdriver.Chrome(options=options)

url = "https://www.scrapethissite.com/pages/ajax-javascript/"
driver.get(url)

years = [2010, 2011, 2012, 2013, 2014, 2015]
all_data = []

for year in years:
    print(f"🔍 Scraping data untuk tahun {year}...")
    
    # Klik tombol tahun
    year_button = WebDriverWait(driver, 10).until(
        EC.element_to_be_clickable((By.LINK_TEXT, str(year)))
    )
    year_button.click()
    
    # Tunggu data dimuat
    time.sleep(3)
    
    # Parse data yang sudah dimuat
    soup = BeautifulSoup(driver.page_source, "html.parser")
    movies = soup.find_all("tr", class_="film")
    
    for movie in movies:
        title = movie.find("td", class_="film-title").text.strip()
        nominations = int(movie.find("td", class_="film-nominations").text.strip())
        awards = int(movie.find("td", class_="film-awards").text.strip())
        best_picture = movie.find("td", class_="film-best-picture").text.strip() == "Yes"
        
        all_data.append([title, nominations, awards, best_picture])

driver.quit()

from selenium import webdriver
from selenium.webdriver.common.by import By
import time
from bs4 import BeautifulSoup

# Setup WebDriver
driver = webdriver.Chrome()
URL = "https://www.scrapethissite.com/pages/frames/"
driver.get(URL)
time.sleep(3)

# 1. Temukan iframe
iframe = driver.find_element(By.TAG_NAME, "iframe")

# 2. Pindah ke dalam iframe
driver.switch_to.frame(iframe)
time.sleep(2)

# 3. Scrape data dalam iframe
soup = BeautifulSoup(driver.page_source, "html.parser")
turtles = soup.find_all("div", class_="turtle-family-card")

all_data = []
for turtle in turtles:
    title = turtle.find("h3", class_="family-name")
    title_text = title.get_text(strip=True) if title else "Nama tidak ditemukan"
    
    # Ambil link detail
    learn_more = turtle.find("a", class_="btn btn-default btn-xs")
    if learn_more:
        detail_link = "https://www.scrapethissite.com" + learn_more["href"]
        
        # Navigate ke halaman detail
        driver.get(detail_link)
        time.sleep(3)
        
        detail_soup = BeautifulSoup(driver.page_source, "html.parser")
        desc = detail_soup.find("p", class_="lead")
        desc_text = desc.get_text(strip=True) if desc else "Deskripsi Tidak Ada"
    else:
        desc_text = "Deskripsi Tidak ada"
    
    all_data.append([title_text, desc_text])

# Kembali ke konteks utama
driver.switch_to.default_content()
driver.quit()

if response.status_code != 200:
    print(f"❌ Gagal mengambil halaman, status: {response.status_code}")
    break

time.sleep(3)  # Jangan terlalu cepat request

# Handle empty values
ot_losses = int(ot_losses_text) if ot_losses_text else 0

driver.quit()  # Selalu tutup browser

def save_to_csv(data, filename="output.csv"):
    columns = ["Column1", "Column2", "Column3"]
    df = pd.DataFrame(data, columns=columns)
    df.to_csv(filename, index=False, encoding="utf-8")
    print(f"📂 Data berhasil disimpan ke {filename}!")