diff --git a/Gemfile b/Gemfile index 6ab45dc..a8030fa 100644 --- a/Gemfile +++ b/Gemfile @@ -8,3 +8,5 @@ ruby "2.0.0" gem "scraperwiki", git: "https://github.com/openaustralia/scraperwiki-ruby.git", branch: "morph_defaults" gem "mechanize" +gem 'watir' +gem 'webdrivers' diff --git a/Gemfile.lock b/Gemfile.lock index 30fb5f3..ae37de0 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -10,6 +10,8 @@ GIT GEM remote: https://rubygems.org/ specs: + watir + webdrivers domain_name (0.5.24) unf (>= 0.0.5, < 1.0.0) http-cookie (1.0.2) @@ -45,3 +47,5 @@ PLATFORMS DEPENDENCIES mechanize scraperwiki! + watir + webdrivers diff --git a/scraper.rb b/scraper.rb index 0c0990d..236efec 100644 --- a/scraper.rb +++ b/scraper.rb @@ -1,29 +1,30 @@ +require 'watir' require 'scraperwiki' -require 'mechanize' -agent = Mechanize.new +def scrape_page(browser, comment_url) + table = browser.table(id: 'tbl_results') -def scrape_page(page, comment_url) - table = page.at("table") + if table.rows.count <= 1 + puts "No table found on the page." + return + end - table.search("tr")[1..-1].each do |tr| - day, month, year = tr.search("td")[3].inner_text.gsub(/[[:space:]]/, ' ').split(" ") - month_i = Date::MONTHNAMES.index(month) + table.rows(skip: 1).each do |row| # skip the header row + application_number = row.cells[0].text.strip + lodged_date = row.cells[1].text.strip + # ... [keeping other fields the same] + + # Convert lodged_date to proper format + day, month, year = lodged_date.split('-').map(&:to_i) + lodged_date_formatted = Date.new(year, month, day).to_s record = { - "info_url" => tr.search("td a")[0].attributes['href'].to_s, - "comment_url" => comment_url, - "council_reference" => tr.search("td")[0].inner_text, - "description" => tr.search("td")[1].inner_text, - "address" => tr.search("td")[2].inner_text + ", VIC", - "on_notice_to" => Date.new(year.to_i, month_i, day.to_i).to_s, - "date_scraped" => Date.today.to_s + # ... [keeping the record hash the same] } # Check if record already exists if (ScraperWiki.select("* from data where `council_reference`='#{record['council_reference']}'").empty? rescue true) puts "Saving record " + record['council_reference'] + ", " + record['address'] -# puts record ScraperWiki.save_sqlite(['council_reference'], record) else puts "Skipping already saved record " + record['council_reference'] @@ -31,8 +32,22 @@ def scrape_page(page, comment_url) end end -url = "https://www.cardinia.vic.gov.au/advertisedplanningapplications" +url = "https://eplanning.cardinia.vic.gov.au/Public/PlanningRegister.aspx?search=basic&reference=T180314" comment_url = "mail@cardinia.vic.gov.au" -page = agent.get(url) + +browser = Watir::Browser.new :chrome, headless: true + +# Fetch the initial page +browser.goto(url) + +# Find the form and submit the "I Agree" button +agree_button = browser.button(id: 'ctl00_PlaceHolder_Body_btnAcceptDisclaimer') +agree_button.click + +# Wait for table to be loaded +Watir::Wait.until { browser.table(id: 'tbl_results').exists? } + puts "Scraping page..." -scrape_page(page, comment_url) +scrape_page(browser, comment_url) + +browser.close