README
¶
hawk-go
Port of HawkAPI's cloudscraper
Everything is exposed to give access to modification, this is subject to change.
Installation:
go get github.com/juiced-aio/hawk-go
Example:
import (
"github.com/useflyent/fhttp/cookiejar"
hawk "github.com/juiced-aio/hawk-go"
http "github.com/useflyent/fhttp"
)
// Client has to be from fhttp and up to CloudFlare's standards, this can include ja3 fingerprint/http2 settings.
// Client also will need a cookie jar.
cookieJar, _: = cookiejar.New(nil)
client.Jar = cookieJar
scraper: = hawk.CFInit(client, "YOUR_KEY_HERE", true)
// You will have to create your own function if you want to solve captchas.
scraper.CaptchaFunction = func(originalURL string, siteKey string) (string, error) {
// CaptchaFunction should return the token as a string.
return "", nil
}
req, _ := http.NewRequest("GET", "https://www.nakedcph.com/en/product/9468/nike-sportswear-dunk-low-disrupt-ck6654-001", nil)
req.Header = http.Header{
"sec-ch-ua": {`"Chromium";v="92", " Not A;Brand";v="99", "Google Chrome";v="92"`},
"sec-ch-ua-mobile": {`?0`},
"upgrade-insecure-requests": {`1`},
"user-agent": {`Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.107 Safari/537.36`},
"accept": {`text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9`},
"sec-fetch-site": {`none`},
"sec-fetch-mode": {`navigate`},
"sec-fetch-user": {`?1`},
"sec-fetch-dest": {`document`},
"accept-encoding": {`gzip, deflate`},
"accept-language": {`en-US,en;q=0.9`},
http.HeaderOrderKey: {"sec-ch-ua", "sec-ch-ua-mobile", "upgrade-insecure-requests", "user-agent", "accept", "sec-fetch-site", "sec-fetch-mode", "sec-fetch-user", "sec-fetch-dest", "accept-encoding", "accept-language"},
http.PHeaderOrderKey: {":method", ":authority", ":scheme", ":path"},
}
resp, err := scraper.Do(req)
All of the logic is based off of HawkAPIs cloudscraper.
For most questions I'd make a ticket in HawkAPI's discord server but if needed my discord is Humphreyyyy#0088.
Documentation
¶
Index ¶
- Variables
- func CheckForCaptcha(body string) bool
- func CreateParams(paramsLong map[string]string) string
- func IsFingerprintChallenge(response *fhttp.Response) bool
- func IsNewCaptchaChallenge(response *fhttp.Response) bool
- func IsNewIUAMChallenge(response *fhttp.Response) bool
- func ReadAndCloseBody(respBody io.ReadCloser) ([]byte, error)
- func ReadAndCopyBody(r interface{}) ([]byte, error)
- func ReadAndUnmarshalBody(respBody io.ReadCloser, x interface{}) error
- type PX
- func (px *PX) GetPayload(endpoint int, token ...string) error
- func (px *PX) GetUserAgent() (string, error)
- func (px *PX) PX1Solve() error
- func (px *PX) PX2Solve() error
- func (px *PX) PX15Solve() error
- func (px *PX) PXGoogleSolve() error
- func (px *PX) PXHoldSolve() error
- func (px *PX) ParseGoogleSiteKey() (string, error)
- func (px *PX) ParsePXResponse(captcha ...bool) map[string]string
- func (px *PX) PostPayloadToPX(isGet ...bool) error
- func (px *PX) Reset()
- func (px *PX) SolveGoogle(url string) (SolveResponse, error)
- func (px *PX) SolveHold(url string) (SolveResponse, error)
- func (px *PX) SolveNormal(url string) (SolveResponse, error)
- func (px *PX) UpdateUserAgent(ua string)
- type PXResponse
- type Scraper
- func (scraper *Scraper) ChallengeInitiationPayload() (*http.Response, error)
- func (scraper *Scraper) Do(request *http.Request) (*http.Response, error)
- func (scraper *Scraper) Get(url string) (resp *http.Response, err error)
- func (scraper *Scraper) GetChallengeResult() (*http.Response, error)
- func (scraper *Scraper) GetPage() (*http.Response, error)
- func (scraper *Scraper) GetPayloadFromAPI() (*http.Response, error)
- func (scraper *Scraper) HandleCaptcha() (*http.Response, error)
- func (scraper *Scraper) HandleFinalApi() (*http.Response, error)
- func (scraper *Scraper) HandleLoopError(errFormat string, err error)
- func (scraper *Scraper) HandleRerun() (*http.Response, error)
- func (scraper *Scraper) InitiateCloudflare() (*http.Response, error)
- func (scraper *Scraper) InitiateScript() (*http.Response, error)
- func (scraper *Scraper) Injection(response *http.Response, err error) (*http.Response, error)
- func (scraper *Scraper) Post(url string, contentType string, body io.Reader) (resp *http.Response, err error)
- func (scraper *Scraper) SendMainPayload() (*http.Response, error)
- func (scraper *Scraper) Solve() (*http.Response, error)
- func (scraper *Scraper) SolvePayload() (*http.Response, error)
- func (scraper *Scraper) SubmitCaptcha() (*http.Response, error)
- func (scraper *Scraper) SubmitChallenge() (*http.Response, error)
- func (scraper *Scraper) SubmitFingerprintChallenge() (*http.Response, error)
- type SolveResponse
Constants ¶
This section is empty.
Variables ¶
View Source
var EP_MAPPING = map[int]string{
1: px_1_ep,
2: px_2_ep,
3: px_cap_15_ep,
4: px_cap_hold_ep,
5: px_cap_google_ep,
}
View Source
var SITE_IDS = map[string]string{
"www.hibbett.com": "PXAJDckzHD",
"www.solebox.com": "PXuR63h57Z",
"www.snipes.com": "PX6XNN2xkk",
"www.onygo.com": "PXJ1N025xg",
"www.revolve.com": "PX78VMO82C",
"www.walmart.com": "PXu6b0qd2S",
"www.ssense.com": "PX58Asv359",
}
Functions ¶
func CheckForCaptcha ¶
func CreateParams ¶
func IsFingerprintChallenge ¶
func IsNewCaptchaChallenge ¶
func IsNewIUAMChallenge ¶
func ReadAndCloseBody ¶
func ReadAndCloseBody(respBody io.ReadCloser) ([]byte, error)
func ReadAndCopyBody ¶
func ReadAndUnmarshalBody ¶
func ReadAndUnmarshalBody(respBody io.ReadCloser, x interface{}) error
Types ¶
type PX ¶
type PX struct {
Domain string
PXID string
Client http.Client
Captcha bool
Gcap bool
CurrentPayload string
Delay int64
PXResponse PXResponse
CaptchaSuccess bool
PXEp string
URL string
PxHeaders http.Header
USER_AGENT string
Params map[string]string
MetaPayload map[string]interface{}
URLBase string
GetCaptcha func(domain string) (string, error)
}
func (*PX) GetUserAgent ¶
func (*PX) PXGoogleSolve ¶
func (*PX) PXHoldSolve ¶
func (*PX) ParseGoogleSiteKey ¶
func (*PX) PostPayloadToPX ¶
func (*PX) SolveGoogle ¶
func (px *PX) SolveGoogle(url string) (SolveResponse, error)
func (*PX) SolveNormal ¶
func (px *PX) SolveNormal(url string) (SolveResponse, error)
func (*PX) UpdateUserAgent ¶
type PXResponse ¶
type PXResponse struct {
Do []string `json:"do"`
}
type Scraper ¶
type Scraper struct {
Client fhttp.Client
CaptchaFunction func(originalURL, siteKey string) (string, error)
FingerprintChallenge bool
Script string
InitScript *fhttp.Response
ChallengePayload *fhttp.Response
MainPayloadResponse *fhttp.Response
InitURL string
RequestURL string
CaptchaScript string
ApiDomain string
TimeOut int
ErrorDelay int
InitHeaders fhttp.Header
ChallengeHeaders fhttp.Header
SubmitHeaders fhttp.Header
OriginalRequest *fhttp.Response
Domain string
Debug bool
Key string
AuthParams map[string]string
Md string
Captcha bool
StartTime time.Time
SolveRetries int
SolveMaxRetries int
Result string
Name string
BaseObj string
RequestPass string
RequestR string
TS int
TargetURL string
InitPayloadRetries int
InitPayloadMaxRetries int
KeyStrUriSafe string
InitChallengeRetries int
InitChallengeMaxRetries int
FetchingChallengeRetries int
FetchingChallengeMaxRetries int
SubmitChallengeRetries int
SubmitChallengeMaxRetries int
ChallengeResultRetries int
ChallengeResultMaxRetries int
FinalApi apiResponse
SubmitFinalChallengeRetries int
SubmitFinalChallengeMaxRetries int
RerunRetries int
RerunMaxRetries int
CaptchaRetries int
CaptchaMaxRetries int
FirstCaptchaResult apiResponse
CaptchaResponseAPI apiResponse
SubmitCaptchaRetries int
SubmitCaptchaMaxRetries int
}
Cloudflare
func (*Scraper) ChallengeInitiationPayload ¶
func (*Scraper) GetChallengeResult ¶
func (*Scraper) GetPayloadFromAPI ¶
func (*Scraper) HandleCaptcha ¶
In Progress
func (*Scraper) HandleLoopError ¶
func (*Scraper) InitiateCloudflare ¶
type SolveResponse ¶
Source Files
Click to show internal directories.
Click to hide internal directories.