Merge pull request #61 from projectdiscovery/feature-raw-requests

HTTP + DNS Raw requests

go.mod

@@ -10,7 +10,6 @@ require (
 	github.com/projectdiscovery/gologger v1.0.0
 	github.com/projectdiscovery/retryabledns v1.0.4
 	github.com/projectdiscovery/retryablehttp-go v1.0.1
-	github.com/valyala/fasttemplate v1.1.0
 	golang.org/x/net v0.0.0-20200425230154-ff2c4b7c35a0
 	gopkg.in/yaml.v2 v2.2.8
 )

go.sum

@@ -19,10 +19,6 @@ github.com/projectdiscovery/retryablehttp-go v1.0.1 h1:V7wUvsZNq1Rcz7+IlcyoyQlNw
 github.com/projectdiscovery/retryablehttp-go v1.0.1/go.mod h1:SrN6iLZilNG1X4neq1D+SBxoqfAF4nyzvmevkTkWsek=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
-github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
-github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
-github.com/valyala/fasttemplate v1.1.0 h1:RZqt0yGBsps8NGvLSGW804QQqCUYYLsaOjTVHy1Ocw4=
-github.com/valyala/fasttemplate v1.1.0/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=

pkg/requests/dns-request.go

@@ -6,7 +6,6 @@ import (
 	"github.com/miekg/dns"
 	"github.com/projectdiscovery/nuclei/pkg/extractors"
 	"github.com/projectdiscovery/nuclei/pkg/matchers"
-	"github.com/valyala/fasttemplate"
 )
 
 // DNSRequest contains a request to be made from a template
@@ -17,6 +16,8 @@ type DNSRequest struct {
 	Type    string `yaml:"type"`
 	Class   string `yaml:"class"`
 	Retries int    `yaml:"retries"`
+	// Raw contains a raw request
+	Raw string `yaml:"raw,omitempty"`
 
 	// Matchers contains the detection mechanism for the request to identify
 	// whether the request was successful
@@ -52,8 +53,9 @@ func (r *DNSRequest) MakeDNSRequest(domain string) (*dns.Msg, error) {
 
 	var q dns.Question
 
-	t := fasttemplate.New(r.Name, "{{", "}}")
-	q.Name = dns.Fqdn(t.ExecuteString(map[string]interface{}{"FQDN": domain}))
+	replacer := newReplacer(map[string]interface{}{"FQDN": domain})
+
+	q.Name = dns.Fqdn(replacer.Replace(r.Name))
 	q.Qclass = toQClass(r.Class)
 	q.Qtype = toQType(r.Type)
 

pkg/requests/http-request.go

@@ -1,6 +1,8 @@
 package requests
 
 import (
+	"bufio"
+	"fmt"
 	"io/ioutil"
 	"net/http"
 	"net/url"
@@ -9,7 +11,6 @@ import (
 	"github.com/projectdiscovery/nuclei/pkg/extractors"
 	"github.com/projectdiscovery/nuclei/pkg/matchers"
 	retryablehttp "github.com/projectdiscovery/retryablehttp-go"
-	"github.com/valyala/fasttemplate"
 )
 
 // HTTPRequest contains a request to be made from a template
@@ -37,6 +38,8 @@ type HTTPRequest struct {
 	Redirects bool `yaml:"redirects,omitempty"`
 	// MaxRedirects is the maximum number of redirects that should be followed.
 	MaxRedirects int `yaml:"max-redirects,omitempty"`
+	// Raw contains raw requests
+	Raw []string `yaml:"raw,omitempty"`
 }
 
 // GetMatchersCondition returns the condition for the matcher
@@ -49,58 +52,79 @@ func (r *HTTPRequest) SetMatchersCondition(condition matchers.ConditionType) {
 	r.matchersCondition = condition
 }
 
-// MakeHTTPRequest creates a *http.Request from a request template
+// MakeHTTPRequest creates a *http.Request from a request configuration
 func (r *HTTPRequest) MakeHTTPRequest(baseURL string) ([]*retryablehttp.Request, error) {
 	parsed, err := url.Parse(baseURL)
 	if err != nil {
 		return nil, err
 	}
 	hostname := parsed.Hostname()
 
-	requests := make([]*retryablehttp.Request, 0, len(r.Path))
+	values := map[string]interface{}{
+		"BaseURL":  baseURL,
+		"Hostname": hostname,
+	}
+
+	if len(r.Raw) > 0 {
+		return r.makeHTTPRequestFromRaw(baseURL, values)
+	}
+
+	return r.makeHTTPRequestFromModel(baseURL, values)
+}
+
+// MakeHTTPRequestFromModel creates a *http.Request from a request template
+func (r *HTTPRequest) makeHTTPRequestFromModel(baseURL string, values map[string]interface{}) (requests []*retryablehttp.Request, err error) {
+	replacer := newReplacer(values)
 	for _, path := range r.Path {
 		// Replace the dynamic variables in the URL if any
-		t := fasttemplate.New(path, "{{", "}}")
-		url := t.ExecuteString(map[string]interface{}{
-			"BaseURL":  baseURL,
-			"Hostname": hostname,
-		})
+		URL := replacer.Replace(path)
 
 		// Build a request on the specified URL
-		req, err := http.NewRequest(r.Method, url, nil)
+		req, err := http.NewRequest(r.Method, URL, nil)
 		if err != nil {
 			return nil, err
 		}
 
-		// Check if the user requested a request body
-		if r.Body != "" {
-			req.Body = ioutil.NopCloser(strings.NewReader(r.Body))
+		request, err := r.fillRequest(req, values)
+		if err != nil {
+			return nil, err
 		}
 
-		// Set the header values requested
-		for header, value := range r.Headers {
-			t := fasttemplate.New(value, "{{", "}}")
-			val := t.ExecuteString(map[string]interface{}{
-				"BaseURL":  baseURL,
-				"Hostname": hostname,
-			})
-			req.Header.Set(header, val)
-		}
+		requests = append(requests, request)
+	}
 
-		// Set some headers only if the header wasn't supplied by the user
-		if _, ok := r.Headers["User-Agent"]; !ok {
-			req.Header.Set("User-Agent", "Nuclei (@pdiscoveryio)")
-		}
-		if _, ok := r.Headers["Accept"]; !ok {
-			req.Header.Set("Accept", "*/*")
+	return
+}
+
+// makeHTTPRequestFromRaw creates a *http.Request from a raw request
+func (r *HTTPRequest) makeHTTPRequestFromRaw(baseURL string, values map[string]interface{}) (requests []*retryablehttp.Request, err error) {
+	replacer := newReplacer(values)
+	for _, raw := range r.Raw {
+		// Add trailing line
+		raw += "\n"
+
+		// Replace the dynamic variables in the URL if any
+		raw = replacer.Replace(raw)
+
+		// Build a parsed request from raw
+		parsedReq, err := http.ReadRequest(bufio.NewReader(strings.NewReader(raw)))
+		if err != nil {
+			return nil, err
 		}
-		if _, ok := r.Headers["Accept-Language"]; !ok {
-			req.Header.Set("Accept-Language", "en")
+
+		// requests generated from http.ReadRequest have incorrect RequestURI, so they
+		// cannot be used to perform another request directly, we need to generate a new one
+		// with the new target url
+		finalURL := fmt.Sprintf("%s%s", baseURL, parsedReq.URL)
+		req, err := http.NewRequest(r.Method, finalURL, parsedReq.Body)
+		if err != nil {
+			return nil, err
 		}
-		req.Header.Set("Connection", "close")
-		req.Close = true
 
-		request, err := retryablehttp.FromRequest(req)
+		// copy headers
+		req.Header = parsedReq.Header.Clone()
+
+		request, err := r.fillRequest(req, values)
 		if err != nil {
 			return nil, err
 		}
@@ -110,3 +134,32 @@ func (r *HTTPRequest) MakeHTTPRequest(baseURL string) ([]*retryablehttp.Request,
 
 	return requests, nil
 }
+
+func (r *HTTPRequest) fillRequest(req *http.Request, values map[string]interface{}) (*retryablehttp.Request, error) {
+	replacer := newReplacer(values)
+	// Check if the user requested a request body
+	if r.Body != "" {
+		req.Body = ioutil.NopCloser(strings.NewReader(r.Body))
+	}
+
+	// Set the header values requested
+	for header, value := range r.Headers {
+		req.Header.Set(header, replacer.Replace(value))
+	}
+
+	// Set some headers only if the header wasn't supplied by the user
+	if _, ok := req.Header["User-Agent"]; !ok {
+		req.Header.Set("User-Agent", "Nuclei (@pdiscoveryio)")
+	}
+
+	if _, ok := req.Header["Accept"]; !ok {
+		req.Header.Set("Accept", "*/*")
+	}
+	if _, ok := req.Header["Accept-Language"]; !ok {
+		req.Header.Set("Accept-Language", "en")
+	}
+	req.Header.Set("Connection", "close")
+	req.Close = true
+
+	return retryablehttp.FromRequest(req)
+}

pkg/requests/util.go

@@ -0,0 +1,16 @@
+package requests
+
+import (
+	"fmt"
+	"strings"
+)
+
+func newReplacer(values map[string]interface{}) *strings.Replacer {
+	var replacerItems []string
+	for k, v := range values {
+		replacerItems = append(replacerItems, fmt.Sprintf("{{%s}}", k))
+		replacerItems = append(replacerItems, fmt.Sprintf("%s", v))
+	}
+
+	return strings.NewReplacer(replacerItems...)
+}