From d62e004a8df8d5cc3f506f3ee8e9867d11b40b21 Mon Sep 17 00:00:00 2001 From: shavit Date: Sat, 27 May 2017 22:28:09 -0400 Subject: [PATCH 1/5] Work with Tor --- browser/browser.go | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/browser/browser.go b/browser/browser.go index 144dfca..2ed9c51 100644 --- a/browser/browser.go +++ b/browser/browser.go @@ -13,6 +13,8 @@ import ( "strings" "time" + "golang.org/x/net/proxy" + "github.com/PuerkitoBio/goquery" "github.com/headzoo/surf/errors" "github.com/headzoo/surf/jar" @@ -90,6 +92,9 @@ type Browsable interface { // SetTransport sets the http library transport mechanism for each request. SetTransport(rt http.RoundTripper) + // Tor sets the proxy url for Tor + Tor(url *url.URL) (err error) + // AddRequestHeader adds a header the browser sends with each request. AddRequestHeader(name, value string) @@ -531,6 +536,17 @@ func (bow *Browser) SetTransport(rt http.RoundTripper) { bow.client.Transport = rt } +// Tor sets the proxy url for Tor +func (bow *Browser) Tor(url *url.URL) (err error) { + dialer, err := proxy.FromURL(url, proxy.Direct) + if err != nil { + return err + } + bow.SetTransport(&http.Transport{Dial: dialer.Dial}) + + return err +} + // AddRequestHeader sets a header the browser sends with each request. func (bow *Browser) AddRequestHeader(name, value string) { bow.headers.Set(name, value) From f17ee06bc65e67c3dab39b275d31dd9faf233f52 Mon Sep 17 00:00:00 2001 From: shavit Date: Sun, 30 Jul 2017 23:03:18 -0400 Subject: [PATCH 2/5] renamed Tor to SetProxy(*url.URL), wrote a unit test --- browser/browser.go | 8 ++++---- browser/browser_test.go | 15 +++++++++++++++ 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/browser/browser.go b/browser/browser.go index 2ed9c51..3d29535 100644 --- a/browser/browser.go +++ b/browser/browser.go @@ -92,8 +92,8 @@ type Browsable interface { // SetTransport sets the http library transport mechanism for each request. SetTransport(rt http.RoundTripper) - // Tor sets the proxy url for Tor - Tor(url *url.URL) (err error) + // Set a proxy URL. You can use it with Tor + SetProxy(url *url.URL) (err error) // AddRequestHeader adds a header the browser sends with each request. AddRequestHeader(name, value string) @@ -536,8 +536,8 @@ func (bow *Browser) SetTransport(rt http.RoundTripper) { bow.client.Transport = rt } -// Tor sets the proxy url for Tor -func (bow *Browser) Tor(url *url.URL) (err error) { +// Set a proxy url for Tor +func (bow *Browser) SetProxy(url *url.URL) (err error) { dialer, err := proxy.FromURL(url, proxy.Direct) if err != nil { return err diff --git a/browser/browser_test.go b/browser/browser_test.go index 20bf899..648a16c 100644 --- a/browser/browser_test.go +++ b/browser/browser_test.go @@ -4,6 +4,7 @@ import ( "io" "net/http" "net/http/httptest" + "net/url" "testing" "time" @@ -139,3 +140,17 @@ func TestCookieHeader(t *testing.T) { t.Errorf("got %d calls, want %d", calls, want) } } + +// Test proxy +// https://github.com/headzoo/surf/pull/56 +func TestSetProxyWillSetTransport(t *testing.T){ + b := newDefaultTestBrowser() + u, err := url.Parse("socks5://127.0.0.1:9050") + if err != nil { + t.Fatal(err) + } + b.SetProxy(u) + if b.transport == nil { + t.Errorf("no transport method") + } +} From 28214e72ed7a05ac76ad67ba29e4e32b1bfbd7e6 Mon Sep 17 00:00:00 2001 From: shavit Date: Sun, 30 Jul 2017 23:13:45 -0400 Subject: [PATCH 3/5] proxy test update --- browser/browser_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/browser/browser_test.go b/browser/browser_test.go index 648a16c..b2490f3 100644 --- a/browser/browser_test.go +++ b/browser/browser_test.go @@ -150,7 +150,7 @@ func TestSetProxyWillSetTransport(t *testing.T){ t.Fatal(err) } b.SetProxy(u) - if b.transport == nil { + if b.client.Transport == nil { t.Errorf("no transport method") } } From fdef6723a80270da110810eb896284de842422b2 Mon Sep 17 00:00:00 2001 From: shavit Date: Mon, 31 Jul 2017 06:45:24 -0400 Subject: [PATCH 4/5] passing a string instead of *url.URL to SetProxy --- browser/browser.go | 10 +++++++--- browser/browser_test.go | 7 +------ 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/browser/browser.go b/browser/browser.go index 3d29535..a8c83ee 100644 --- a/browser/browser.go +++ b/browser/browser.go @@ -93,7 +93,7 @@ type Browsable interface { SetTransport(rt http.RoundTripper) // Set a proxy URL. You can use it with Tor - SetProxy(url *url.URL) (err error) + SetProxy(u string) (err error) // AddRequestHeader adds a header the browser sends with each request. AddRequestHeader(name, value string) @@ -537,8 +537,12 @@ func (bow *Browser) SetTransport(rt http.RoundTripper) { } // Set a proxy url for Tor -func (bow *Browser) SetProxy(url *url.URL) (err error) { - dialer, err := proxy.FromURL(url, proxy.Direct) +func (bow *Browser) SetProxy(u string) (err error) { + _url, err := url.Parse(u) + if err != nil { + return err + } + dialer, err := proxy.FromURL(_url, proxy.Direct) if err != nil { return err } diff --git a/browser/browser_test.go b/browser/browser_test.go index b2490f3..9c99ac2 100644 --- a/browser/browser_test.go +++ b/browser/browser_test.go @@ -4,7 +4,6 @@ import ( "io" "net/http" "net/http/httptest" - "net/url" "testing" "time" @@ -145,11 +144,7 @@ func TestCookieHeader(t *testing.T) { // https://github.com/headzoo/surf/pull/56 func TestSetProxyWillSetTransport(t *testing.T){ b := newDefaultTestBrowser() - u, err := url.Parse("socks5://127.0.0.1:9050") - if err != nil { - t.Fatal(err) - } - b.SetProxy(u) + b.SetProxy("socks5://127.0.0.1:9050") if b.client.Transport == nil { t.Errorf("no transport method") } From 89896666e11590a5fa4054b96ac6c235465b5faf Mon Sep 17 00:00:00 2001 From: lxt2 Date: Sat, 5 Aug 2017 19:50:21 +1000 Subject: [PATCH 5/5] Add shavit to the README --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 99851aa..e94219b 100644 --- a/README.md +++ b/README.md @@ -75,6 +75,7 @@ Contributions have been made to Surf by the following awesome developers: * [nicot](https://github.com/nicot) * [Joseph Watson](https://github.com/jtwatson) * [lxt2](https://github.com/lxt2) +* [shavit](https://github.com/shavit) The idea to create Surf was born in [this Reddit thread](http://www.reddit.com/r/golang/comments/2efw1q/mechanize_in_go/cjz4lze).