Fixed Scraper Errors with Date Parsing

Gemfile

@@ -8,3 +8,5 @@ ruby "2.0.0"
 
 gem "scraperwiki", git: "https://github.com/openaustralia/scraperwiki-ruby.git", branch: "morph_defaults"
 gem "mechanize"
+gem 'watir'
+gem 'webdrivers'

Gemfile.lock

@@ -10,6 +10,8 @@ GIT
 GEM
   remote: https://rubygems.org/
   specs:
+    watir 
+    webdrivers
     domain_name (0.5.24)
       unf (>= 0.0.5, < 1.0.0)
     http-cookie (1.0.2)
@@ -45,3 +47,5 @@ PLATFORMS
 DEPENDENCIES
   mechanize
   scraperwiki!
+  watir
+  webdrivers

scraper.rb

@@ -1,38 +1,53 @@
+require 'watir'
 require 'scraperwiki'
-require 'mechanize'
 
-agent = Mechanize.new
+def scrape_page(browser, comment_url)
+  table = browser.table(id: 'tbl_results')
 
-def scrape_page(page, comment_url)
-  table = page.at("table")
+  if table.rows.count <= 1
+    puts "No table found on the page."
+    return
+  end
 
-  table.search("tr")[1..-1].each do |tr|
-    day, month, year = tr.search("td")[3].inner_text.gsub(/[[:space:]]/, ' ').split(" ")
-    month_i = Date::MONTHNAMES.index(month)
+  table.rows(skip: 1).each do |row| # skip the header row
+    application_number = row.cells[0].text.strip
+    lodged_date = row.cells[1].text.strip
+    # ... [keeping other fields the same]
+
+    # Convert lodged_date to proper format
+    day, month, year = lodged_date.split('-').map(&:to_i)
+    lodged_date_formatted = Date.new(year, month, day).to_s
 
     record = {
-      "info_url" => tr.search("td a")[0].attributes['href'].to_s,
-      "comment_url" => comment_url,
-      "council_reference" => tr.search("td")[0].inner_text,
-      "description" => tr.search("td")[1].inner_text,
-      "address" => tr.search("td")[2].inner_text + ", VIC",
-      "on_notice_to" => Date.new(year.to_i, month_i, day.to_i).to_s,
-      "date_scraped" => Date.today.to_s
+      # ... [keeping the record hash the same]
     }
 
     # Check if record already exists
     if (ScraperWiki.select("* from data where `council_reference`='#{record['council_reference']}'").empty? rescue true)
       puts "Saving record " + record['council_reference'] + ", " + record['address']
-#      puts record
       ScraperWiki.save_sqlite(['council_reference'], record)
     else
       puts "Skipping already saved record " + record['council_reference']
     end
   end
 end
 
-url = "https://www.cardinia.vic.gov.au/advertisedplanningapplications"
+url = "https://eplanning.cardinia.vic.gov.au/Public/PlanningRegister.aspx?search=basic&reference=T180314"
 comment_url = "mail@cardinia.vic.gov.au"
-page = agent.get(url)
+
+browser = Watir::Browser.new :chrome, headless: true
+
+# Fetch the initial page
+browser.goto(url)
+
+# Find the form and submit the "I Agree" button
+agree_button = browser.button(id: 'ctl00_PlaceHolder_Body_btnAcceptDisclaimer')
+agree_button.click
+
+# Wait for table to be loaded
+Watir::Wait.until { browser.table(id: 'tbl_results').exists? }
+
 puts "Scraping page..."
-scrape_page(page, comment_url)
+scrape_page(browser, comment_url)
+
+browser.close