From 385149e0cde200eaaa830cbb6162632cdc56ca05 Mon Sep 17 00:00:00 2001
From: Jamas Alberta <104399013+jamasalbertastash@users.noreply.github.com>
Date: Wed, 9 Aug 2023 10:41:22 +1000
Subject: [PATCH 01/11] date error scraper.rb

Add a guard clause: Before the Date creation, add a guard clause to skip any invalid date values.
---
 scraper.rb | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/scraper.rb b/scraper.rb
index 0c0990d..eb549bf 100644
--- a/scraper.rb
+++ b/scraper.rb
@@ -8,8 +8,18 @@ def scrape_page(page, comment_url)
 
   table.search("tr")[1..-1].each do |tr|
     day, month, year = tr.search("td")[3].inner_text.gsub(/[[:space:]]/, ' ').split(" ")
+    
+    # Print the date values for debugging
+    puts "Parsed Date: Day: #{day}, Month: #{month}, Year: #{year}"
+
     month_i = Date::MONTHNAMES.index(month)
 
+    # Guard clause
+    unless day && month_i && year
+      puts "Invalid date values. Skipping..."
+      next
+    end
+
     record = {
       "info_url" => tr.search("td a")[0].attributes['href'].to_s,
       "comment_url" => comment_url,

From 5706a218cfa41d5e01a50b07cf37d7dd05027853 Mon Sep 17 00:00:00 2001
From: Jamas Alberta <104399013+jamasalbertastash@users.noreply.github.com>
Date: Wed, 9 Aug 2023 10:47:32 +1000
Subject: [PATCH 02/11] Update scraper.rb to fix dates DDMMYY

---
 scraper.rb | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/scraper.rb b/scraper.rb
index eb549bf..9768a38 100644
--- a/scraper.rb
+++ b/scraper.rb
@@ -7,33 +7,36 @@ def scrape_page(page, comment_url)
   table = page.at("table")
 
   table.search("tr")[1..-1].each do |tr|
-    day, month, year = tr.search("td")[3].inner_text.gsub(/[[:space:]]/, ' ').split(" ")
+    date_str = tr.search("td")[3].inner_text.gsub(/[[:space:]]/, ' ')
+    
+    # Split the date string on periods
+    day, month, year = date_str.split(".")
     
     # Print the date values for debugging
     puts "Parsed Date: Day: #{day}, Month: #{month}, Year: #{year}"
 
-    month_i = Date::MONTHNAMES.index(month)
-
     # Guard clause
-    unless day && month_i && year
+    unless day && month && year
       puts "Invalid date values. Skipping..."
       next
     end
 
+    # Adjust the year to YYYY format
+    year = "20" + year if year.length == 2
+
     record = {
       "info_url" => tr.search("td a")[0].attributes['href'].to_s,
       "comment_url" => comment_url,
       "council_reference" => tr.search("td")[0].inner_text,
       "description" => tr.search("td")[1].inner_text,
       "address" => tr.search("td")[2].inner_text + ", VIC",
-      "on_notice_to" => Date.new(year.to_i, month_i, day.to_i).to_s,
+      "on_notice_to" => Date.new(year.to_i, month.to_i, day.to_i).to_s,
       "date_scraped" => Date.today.to_s
     }
 
     # Check if record already exists
     if (ScraperWiki.select("* from data where `council_reference`='#{record['council_reference']}'").empty? rescue true)
       puts "Saving record " + record['council_reference'] + ", " + record['address']
-#      puts record
       ScraperWiki.save_sqlite(['council_reference'], record)
     else
       puts "Skipping already saved record " + record['council_reference']

From 61c99434f78c822fd112a8811fe42ca4f07d670c Mon Sep 17 00:00:00 2001
From: Jamas Alberta <104399013+jamasalbertastash@users.noreply.github.com>
Date: Thu, 10 Aug 2023 09:32:12 +1000
Subject: [PATCH 03/11] Update scraper.rb

---
 scraper.rb | 41 ++++++++++++++++++-----------------------
 1 file changed, 18 insertions(+), 23 deletions(-)

diff --git a/scraper.rb b/scraper.rb
index 9768a38..4796a01 100644
--- a/scraper.rb
+++ b/scraper.rb
@@ -4,33 +4,28 @@
 agent = Mechanize.new
 
 def scrape_page(page, comment_url)
-  table = page.at("table")
+  table = page.at("#tbl_results")
 
-  table.search("tr")[1..-1].each do |tr|
-    date_str = tr.search("td")[3].inner_text.gsub(/[[:space:]]/, ' ')
-    
-    # Split the date string on periods
-    day, month, year = date_str.split(".")
-    
-    # Print the date values for debugging
-    puts "Parsed Date: Day: #{day}, Month: #{month}, Year: #{year}"
+  table.search("tbody tr").each do |tr|
+    application_number = tr.search("td")[0].inner_text.strip
+    lodged_date = tr.search("td")[1].inner_text.strip
+    decision_date = tr.search("td")[2].inner_text.strip
+    address = tr.search("td")[3].inner_text.strip
+    reason_for_permit = tr.search("td")[4].inner_text.strip
+    ward = tr.search("td")[5].inner_text.strip
+    status = tr.search("td")[6].inner_text.strip
 
-    # Guard clause
-    unless day && month && year
-      puts "Invalid date values. Skipping..."
-      next
-    end
-
-    # Adjust the year to YYYY format
-    year = "20" + year if year.length == 2
+    # Convert lodged_date to proper format
+    day, month, year = lodged_date.split('-').map(&:to_i)
+    lodged_date_formatted = Date.new(year, month, day).to_s
 
     record = {
-      "info_url" => tr.search("td a")[0].attributes['href'].to_s,
+      "info_url" => nil,  # No detail page URL provided in the given snippet.
       "comment_url" => comment_url,
-      "council_reference" => tr.search("td")[0].inner_text,
-      "description" => tr.search("td")[1].inner_text,
-      "address" => tr.search("td")[2].inner_text + ", VIC",
-      "on_notice_to" => Date.new(year.to_i, month.to_i, day.to_i).to_s,
+      "council_reference" => application_number,
+      "description" => reason_for_permit,
+      "address" => address,
+      "on_notice_to" => lodged_date_formatted,
       "date_scraped" => Date.today.to_s
     }
 
@@ -44,7 +39,7 @@ def scrape_page(page, comment_url)
   end
 end
 
-url = "https://www.cardinia.vic.gov.au/advertisedplanningapplications"
+url = "https://eplanning.cardinia.vic.gov.au/Public/PlanningRegister.aspx?search=basic&reference=T180314"
 comment_url = "mail@cardinia.vic.gov.au"
 page = agent.get(url)
 puts "Scraping page..."

From 93bc9ffbf9644bb60de6da6b98c1a4198b2ee0a9 Mon Sep 17 00:00:00 2001
From: Jamas Alberta <104399013+jamasalbertastash@users.noreply.github.com>
Date: Thu, 10 Aug 2023 09:42:31 +1000
Subject: [PATCH 04/11] Update scraper.rb

---
 scraper.rb | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/scraper.rb b/scraper.rb
index 4796a01..998ce9f 100644
--- a/scraper.rb
+++ b/scraper.rb
@@ -4,6 +4,10 @@
 agent = Mechanize.new
 
 def scrape_page(page, comment_url)
+  if table.nil?
+  puts "No table found on the page."
+  return
+end
   table = page.at("#tbl_results")
 
   table.search("tbody tr").each do |tr|

From 47f8dc80856435a20c8c763aa230c881c09ca3b9 Mon Sep 17 00:00:00 2001
From: Jamas Alberta <104399013+jamasalbertastash@users.noreply.github.com>
Date: Thu, 10 Aug 2023 09:46:40 +1000
Subject: [PATCH 05/11] Update scraper.rb

---
 scraper.rb | 17 +++++++++++++----
 1 file changed, 13 insertions(+), 4 deletions(-)

diff --git a/scraper.rb b/scraper.rb
index 998ce9f..ff209f9 100644
--- a/scraper.rb
+++ b/scraper.rb
@@ -3,13 +3,22 @@
 
 agent = Mechanize.new
 
+url = "https://eplanning.cardinia.vic.gov.au/Public/PlanningRegister.aspx?search=basic&reference=T180314"
+page = agent.get(url)
+
+# Find the "I Agree" button and submit the form
+agree_button = page.at("input#ctl00_PlaceHolder_Body_btnAcceptDisclaimer")
+page = agree_button.click
+
+# Continue with scraping as usual
 def scrape_page(page, comment_url)
-  if table.nil?
-  puts "No table found on the page."
-  return
-end
   table = page.at("#tbl_results")
 
+  if table.nil?
+    puts "No table found on the page."
+    return
+  end
+
   table.search("tbody tr").each do |tr|
     application_number = tr.search("td")[0].inner_text.strip
     lodged_date = tr.search("td")[1].inner_text.strip

From 10590dc6f26c6c8c5ac4c1a4260cded45fd71dc2 Mon Sep 17 00:00:00 2001
From: Jamas Alberta <104399013+jamasalbertastash@users.noreply.github.com>
Date: Thu, 10 Aug 2023 09:48:17 +1000
Subject: [PATCH 06/11] Update scraper.rb

---
 scraper.rb | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/scraper.rb b/scraper.rb
index ff209f9..57e2d99 100644
--- a/scraper.rb
+++ b/scraper.rb
@@ -6,9 +6,10 @@
 url = "https://eplanning.cardinia.vic.gov.au/Public/PlanningRegister.aspx?search=basic&reference=T180314"
 page = agent.get(url)
 
-# Find the "I Agree" button and submit the form
-agree_button = page.at("input#ctl00_PlaceHolder_Body_btnAcceptDisclaimer")
-page = agree_button.click
+# Find the form and then the "I Agree" button and submit it
+form = agent.page.form_with(id: "aspnetForm")
+agree_button = form.button_with(id: "ctl00_PlaceHolder_Body_btnAcceptDisclaimer")
+page = agent.submit(form, agree_button)
 
 # Continue with scraping as usual
 def scrape_page(page, comment_url)

From c7308ad295ac385efe48b4dc3580068d75a15de8 Mon Sep 17 00:00:00 2001
From: Jamas Alberta <104399013+jamasalbertastash@users.noreply.github.com>
Date: Thu, 10 Aug 2023 10:10:13 +1000
Subject: [PATCH 07/11] Update scraper.rb

---
 scraper.rb | 39 +++++++++++++++------------------------
 1 file changed, 15 insertions(+), 24 deletions(-)

diff --git a/scraper.rb b/scraper.rb
index 57e2d99..3bbc763 100644
--- a/scraper.rb
+++ b/scraper.rb
@@ -1,17 +1,6 @@
 require 'scraperwiki'
 require 'mechanize'
 
-agent = Mechanize.new
-
-url = "https://eplanning.cardinia.vic.gov.au/Public/PlanningRegister.aspx?search=basic&reference=T180314"
-page = agent.get(url)
-
-# Find the form and then the "I Agree" button and submit it
-form = agent.page.form_with(id: "aspnetForm")
-agree_button = form.button_with(id: "ctl00_PlaceHolder_Body_btnAcceptDisclaimer")
-page = agent.submit(form, agree_button)
-
-# Continue with scraping as usual
 def scrape_page(page, comment_url)
   table = page.at("#tbl_results")
 
@@ -23,24 +12,14 @@ def scrape_page(page, comment_url)
   table.search("tbody tr").each do |tr|
     application_number = tr.search("td")[0].inner_text.strip
     lodged_date = tr.search("td")[1].inner_text.strip
-    decision_date = tr.search("td")[2].inner_text.strip
-    address = tr.search("td")[3].inner_text.strip
-    reason_for_permit = tr.search("td")[4].inner_text.strip
-    ward = tr.search("td")[5].inner_text.strip
-    status = tr.search("td")[6].inner_text.strip
-
+    # ... [keeping other fields the same]
+    
     # Convert lodged_date to proper format
     day, month, year = lodged_date.split('-').map(&:to_i)
     lodged_date_formatted = Date.new(year, month, day).to_s
 
     record = {
-      "info_url" => nil,  # No detail page URL provided in the given snippet.
-      "comment_url" => comment_url,
-      "council_reference" => application_number,
-      "description" => reason_for_permit,
-      "address" => address,
-      "on_notice_to" => lodged_date_formatted,
-      "date_scraped" => Date.today.to_s
+      # ... [keeping the record hash the same]
     }
 
     # Check if record already exists
@@ -53,8 +32,20 @@ def scrape_page(page, comment_url)
   end
 end
 
+agent = Mechanize.new
+
 url = "https://eplanning.cardinia.vic.gov.au/Public/PlanningRegister.aspx?search=basic&reference=T180314"
 comment_url = "mail@cardinia.vic.gov.au"
+
+# Fetch the initial page
+page = agent.get(url)
+
+# Find the form and submit the "I Agree" button
+form = agent.page.form_with(id: "aspnetForm")
+agree_button = form.button_with(id: "ctl00_PlaceHolder_Body_btnAcceptDisclaimer")
+page = agent.submit(form, agree_button)
+
+# Now that you've agreed, re-fetch the page containing the data
 page = agent.get(url)
 puts "Scraping page..."
 scrape_page(page, comment_url)

From 53a30073fac1a4775f591eb142a46ff13926de84 Mon Sep 17 00:00:00 2001
From: Jamas Alberta <104399013+jamasalbertastash@users.noreply.github.com>
Date: Thu, 10 Aug 2023 10:19:43 +1000
Subject: [PATCH 08/11] Update Gemfile

---
 Gemfile | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/Gemfile b/Gemfile
index 6ab45dc..a8030fa 100644
--- a/Gemfile
+++ b/Gemfile
@@ -8,3 +8,5 @@ ruby "2.0.0"
 
 gem "scraperwiki", git: "https://github.com/openaustralia/scraperwiki-ruby.git", branch: "morph_defaults"
 gem "mechanize"
+gem 'watir'
+gem 'webdrivers'

From 53618fb9ea9402882b28264f47942197a948bdba Mon Sep 17 00:00:00 2001
From: Jamas Alberta <104399013+jamasalbertastash@users.noreply.github.com>
Date: Thu, 10 Aug 2023 10:20:04 +1000
Subject: [PATCH 09/11] Update scraper.rb

---
 scraper.rb | 34 ++++++++++++++++++----------------
 1 file changed, 18 insertions(+), 16 deletions(-)

diff --git a/scraper.rb b/scraper.rb
index 3bbc763..236efec 100644
--- a/scraper.rb
+++ b/scraper.rb
@@ -1,17 +1,17 @@
+require 'watir'
 require 'scraperwiki'
-require 'mechanize'
 
-def scrape_page(page, comment_url)
-  table = page.at("#tbl_results")
+def scrape_page(browser, comment_url)
+  table = browser.table(id: 'tbl_results')
 
-  if table.nil?
+  if table.rows.count <= 1
     puts "No table found on the page."
     return
   end
 
-  table.search("tbody tr").each do |tr|
-    application_number = tr.search("td")[0].inner_text.strip
-    lodged_date = tr.search("td")[1].inner_text.strip
+  table.rows(skip: 1).each do |row| # skip the header row
+    application_number = row.cells[0].text.strip
+    lodged_date = row.cells[1].text.strip
     # ... [keeping other fields the same]
     
     # Convert lodged_date to proper format
@@ -32,20 +32,22 @@ def scrape_page(page, comment_url)
   end
 end
 
-agent = Mechanize.new
-
 url = "https://eplanning.cardinia.vic.gov.au/Public/PlanningRegister.aspx?search=basic&reference=T180314"
 comment_url = "mail@cardinia.vic.gov.au"
 
+browser = Watir::Browser.new :chrome, headless: true
+
 # Fetch the initial page
-page = agent.get(url)
+browser.goto(url)
 
 # Find the form and submit the "I Agree" button
-form = agent.page.form_with(id: "aspnetForm")
-agree_button = form.button_with(id: "ctl00_PlaceHolder_Body_btnAcceptDisclaimer")
-page = agent.submit(form, agree_button)
+agree_button = browser.button(id: 'ctl00_PlaceHolder_Body_btnAcceptDisclaimer')
+agree_button.click
+
+# Wait for table to be loaded
+Watir::Wait.until { browser.table(id: 'tbl_results').exists? }
 
-# Now that you've agreed, re-fetch the page containing the data
-page = agent.get(url)
 puts "Scraping page..."
-scrape_page(page, comment_url)
+scrape_page(browser, comment_url)
+
+browser.close

From 42be149559834553b16cf770a964c436f4c77c9d Mon Sep 17 00:00:00 2001
From: Jamas Alberta <104399013+jamasalbertastash@users.noreply.github.com>
Date: Thu, 10 Aug 2023 10:29:21 +1000
Subject: [PATCH 10/11] Rename Gemfile.lock to Gemfile_old.lock

---
 Gemfile.lock => Gemfile_old.lock | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename Gemfile.lock => Gemfile_old.lock (100%)

diff --git a/Gemfile.lock b/Gemfile_old.lock
similarity index 100%
rename from Gemfile.lock
rename to Gemfile_old.lock

From 40fb485af61a9cb2f61818a0986c8ff8580d80be Mon Sep 17 00:00:00 2001
From: Jamas Alberta <104399013+jamasalbertastash@users.noreply.github.com>
Date: Thu, 10 Aug 2023 10:33:23 +1000
Subject: [PATCH 11/11] Update and rename Gemfile_old.lock to Gemfile.lock

---
 Gemfile_old.lock => Gemfile.lock | 4 ++++
 1 file changed, 4 insertions(+)
 rename Gemfile_old.lock => Gemfile.lock (95%)

diff --git a/Gemfile_old.lock b/Gemfile.lock
similarity index 95%
rename from Gemfile_old.lock
rename to Gemfile.lock
index 30fb5f3..ae37de0 100644
--- a/Gemfile_old.lock
+++ b/Gemfile.lock
@@ -10,6 +10,8 @@ GIT
 GEM
   remote: https://rubygems.org/
   specs:
+    watir 
+    webdrivers
     domain_name (0.5.24)
       unf (>= 0.0.5, < 1.0.0)
     http-cookie (1.0.2)
@@ -45,3 +47,5 @@ PLATFORMS
 DEPENDENCIES
   mechanize
   scraperwiki!
+  watir
+  webdrivers