From 6707bc777a92bfa6c2a5937a3cbc5a9b07ba7b74 Mon Sep 17 00:00:00 2001 From: Ramana Reddy <90540245+RamanaReddy0M@users.noreply.github.com> Date: Fri, 30 Jun 2023 23:32:00 +0530 Subject: [PATCH] fix showing multiple failure matches per template on -ms set (#3770) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix showing multiple failure matchers per template add integration test * exclude AS134029 from unit test * Add flag for match status per request * chore(deps): bump golangci/golangci-lint-action from 3.4.0 to 3.5.0 (#3777) Bumps [golangci/golangci-lint-action](https://github.com/golangci/golangci-lint-action) from 3.4.0 to 3.5.0. - [Release notes](https://github.com/golangci/golangci-lint-action/releases) - [Commits](https://github.com/golangci/golangci-lint-action/compare/v3.4.0...v3.5.0) --- updated-dependencies: - dependency-name: golangci/golangci-lint-action dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * chore(deps): bump github.com/xanzy/go-gitlab in /v2 (#3778) Bumps [github.com/xanzy/go-gitlab](https://github.com/xanzy/go-gitlab) from 0.83.0 to 0.84.0. - [Changelog](https://github.com/xanzy/go-gitlab/blob/master/releases_test.go) - [Commits](https://github.com/xanzy/go-gitlab/compare/v0.83.0...v0.84.0) --- updated-dependencies: - dependency-name: github.com/xanzy/go-gitlab dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * chore(deps): bump github.com/spf13/cast from 1.5.0 to 1.5.1 in /v2 (#3780) Bumps [github.com/spf13/cast](https://github.com/spf13/cast) from 1.5.0 to 1.5.1. - [Release notes](https://github.com/spf13/cast/releases) - [Commits](https://github.com/spf13/cast/compare/v1.5.0...v1.5.1) --- updated-dependencies: - dependency-name: github.com/spf13/cast dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * enable no-httpx when passive scan is launched (#3789) * chore(deps): bump github.com/projectdiscovery/fastdialer from 0.0.26 to 0.0.28 in /v2 (#3779) * chore(deps): bump github.com/projectdiscovery/fastdialer in /v2 Bumps [github.com/projectdiscovery/fastdialer](https://github.com/projectdiscovery/fastdialer) from 0.0.26 to 0.0.28. - [Release notes](https://github.com/projectdiscovery/fastdialer/releases) - [Commits](https://github.com/projectdiscovery/fastdialer/compare/v0.0.26...v0.0.28) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/fastdialer dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] * Bump retryabledns to 0.28 * Update the retryabledns --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: shubhamrasal * deprecatedProtocolNameTemplates concurrent map writes (#3785) * deprecatedProtocolNameTemplates * use syncLock * fix lint error * change version in deprecated warning msg * comment asnmap expand unit test --------- Co-authored-by: Tarun Koyalwar Co-authored-by: Tarun Koyalwar <45962551+tarunKoyalwar@users.noreply.github.com> * Issue 3339 headless fuzz (#3790) * Basic headless fuzzing * Remove debug statements * Add integration tests * Update template * Fix recognize payload value in matcher * Update tempalte * use req.SetURL() --------- Co-authored-by: Tarun Koyalwar * Auto Generate Syntax Docs + JSONSchema [Fri Jun 9 00:23:32 UTC 2023] :robot: * Add headless header and status matchers (#3794) * add headless header and status matchers * rename headers as header * add integration test for header+status * fix typo * chore(deps): bump golang from 1.20.4-alpine to 1.20.5-alpine (#3809) Bumps golang from 1.20.4-alpine to 1.20.5-alpine. --- updated-dependencies: - dependency-name: golang dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * chore(deps): bump github.com/go-playground/validator/v10 in /v2 (#3810) Bumps [github.com/go-playground/validator/v10](https://github.com/go-playground/validator) from 10.11.2 to 10.14.1. - [Release notes](https://github.com/go-playground/validator/releases) - [Commits](https://github.com/go-playground/validator/compare/v10.11.2...v10.14.1) --- updated-dependencies: - dependency-name: github.com/go-playground/validator/v10 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * chore(deps): bump github.com/projectdiscovery/rawhttp in /v2 (#3811) Bumps [github.com/projectdiscovery/rawhttp](https://github.com/projectdiscovery/rawhttp) from 0.1.11 to 0.1.13. - [Release notes](https://github.com/projectdiscovery/rawhttp/releases) - [Commits](https://github.com/projectdiscovery/rawhttp/compare/v0.1.11...v0.1.13) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/rawhttp dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * chore(deps): bump github.com/go-git/go-git/v5 from 5.6.1 to 5.7.0 in /v2 (#3812) Bumps [github.com/go-git/go-git/v5](https://github.com/go-git/go-git) from 5.6.1 to 5.7.0. - [Release notes](https://github.com/go-git/go-git/releases) - [Commits](https://github.com/go-git/go-git/compare/v5.6.1...v5.7.0) --- updated-dependencies: - dependency-name: github.com/go-git/go-git/v5 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * chore(deps): bump github.com/projectdiscovery/hmap in /v2 (#3781) Bumps [github.com/projectdiscovery/hmap](https://github.com/projectdiscovery/hmap) from 0.0.11 to 0.0.13. - [Release notes](https://github.com/projectdiscovery/hmap/releases) - [Commits](https://github.com/projectdiscovery/hmap/compare/v0.0.11...v0.0.13) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/hmap dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Using safe dereferencing * adding comment * fixing and condition * fixing test id * adding integration test * update goflags dependency * update goflags dependency * bump goflags v0.1.9 => v0.1.10 * handle failure matcher flags logic at executor itself * add integration test to matcher status per request * Adding random tls impersonate (#3844) * adding random tls impersonate * dep update --------- Co-authored-by: sandeep <8293321+ehsandeep@users.noreply.github.com> * Use templateman enhance api to populate CVE info (#3788) * use templateman enhance api to populate cve info * rename cve-annotate => tmc add additional flags to format, lint and enhance template using templateman apis * minior changes * remove duplicate code * misc update * Add validate and error log option * print if updated * print format and enhance only if updated * make max-request optional * fix reference unmarshal error * fix removing self-contained tag --------- Co-authored-by: sandeep <8293321+ehsandeep@users.noreply.github.com> Co-authored-by: Tarun Koyalwar Co-authored-by: Sandeep Singh * fix matcher status with network protocol * fix test * remove -msr flag --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Dogan Can Bakir <65292895+dogancanbakir@users.noreply.github.com> Co-authored-by: shubhamrasal Co-authored-by: 三米前有蕉皮 Co-authored-by: Tarun Koyalwar Co-authored-by: Tarun Koyalwar <45962551+tarunKoyalwar@users.noreply.github.com> Co-authored-by: Shubham Rasal Co-authored-by: GitHub Action Co-authored-by: Mzack9999 Co-authored-by: sandeep <8293321+ehsandeep@users.noreply.github.com> Co-authored-by: Sandeep Singh --- integration_tests/http/matcher-status.yaml | 40 +++++++++++++++ v2/cmd/integration-test/http.go | 19 +++++++ v2/pkg/output/output.go | 53 ++++++++++---------- v2/pkg/protocols/common/executer/executer.go | 22 +++++--- v2/pkg/protocols/network/request.go | 2 + v2/pkg/protocols/network/request_test.go | 2 +- v2/pkg/testutils/integration.go | 2 +- 7 files changed, 106 insertions(+), 34 deletions(-) create mode 100644 integration_tests/http/matcher-status.yaml diff --git a/integration_tests/http/matcher-status.yaml b/integration_tests/http/matcher-status.yaml new file mode 100644 index 0000000000..5704c2a3d9 --- /dev/null +++ b/integration_tests/http/matcher-status.yaml @@ -0,0 +1,40 @@ +id: matchet-status + +info: + name: Test Matcher Status + author: pdteam + severity: critical + +variables: + username: test + password: admin + date: 2023-05-31 + +http: + - method: GET + path: + - "{{RootURL}}/login?username={{username}}&password={{password}}" + - "{{BaseURL}}/admin-pannel" + + - method: GET + path: + - "{{BaseURL}}/dashboard?date={{date}}" + - "{{BaseURL}}/signup" + + - method: POST + path: + - "{{BaseURL}}/filemanager/upload.php" + body: "fldr=&url=file:///etc/passwd" + + + stop-at-first-match: true + matchers-condition: and + matchers: + - type: word + part: body + words: + - "matcher status" + + - type: status + status: + - 200 diff --git a/v2/cmd/integration-test/http.go b/v2/cmd/integration-test/http.go index fa57602caa..9a3046b481 100644 --- a/v2/cmd/integration-test/http.go +++ b/v2/cmd/integration-test/http.go @@ -78,6 +78,7 @@ var httpTestcases = map[string]testutils.TestCase{ "http/cl-body-with-header.yaml": &httpCLBodyWithHeader{}, "http/save-extractor-values-to-file.yaml": &httpSaveExtractorValuesToFile{}, "http/cli-with-constants.yaml": &ConstantWithCliVar{}, + "http/matcher-status.yaml": &matcherStatusTest{}, "http/disable-path-automerge.yaml": &httpDisablePathAutomerge{}, } @@ -1425,6 +1426,24 @@ func (h *ConstantWithCliVar) Execute(filePath string) error { return expectResultsCount(got, 1) } +type matcherStatusTest struct{} + +// Execute executes a test case and returns an error if occurred +func (h *matcherStatusTest) Execute(filePath string) error { + router := httprouter.New() + router.GET("/200", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + w.WriteHeader(http.StatusOK) + }) + ts := httptest.NewServer(router) + defer ts.Close() + + results, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug, "-ms") + if err != nil { + return err + } + return expectResultsCount(results, 1) +} + // disable path automerge in raw request type httpDisablePathAutomerge struct{} diff --git a/v2/pkg/output/output.go b/v2/pkg/output/output.go index 9d2895bc74..a23d6f3812 100644 --- a/v2/pkg/output/output.go +++ b/v2/pkg/output/output.go @@ -45,19 +45,19 @@ type Writer interface { // StandardWriter is a writer writing output to file and screen for results. type StandardWriter struct { - json bool - jsonReqResp bool - timestamp bool - noMetadata bool - matcherStatus bool - mutex *sync.Mutex - aurora aurora.Aurora - outputFile io.WriteCloser - traceFile io.WriteCloser - errorFile io.WriteCloser - severityColors func(severity.Severity) string - storeResponse bool - storeResponseDir string + json bool + jsonReqResp bool + timestamp bool + noMetadata bool + matcherStatus bool + mutex *sync.Mutex + aurora aurora.Aurora + outputFile io.WriteCloser + traceFile io.WriteCloser + errorFile io.WriteCloser + severityColors func(severity.Severity) string + storeResponse bool + storeResponseDir string } var decolorizerRegex = regexp.MustCompile(`\x1B\[[0-9;]*[a-zA-Z]`) @@ -187,20 +187,21 @@ func NewStandardWriter(options *types.Options) (*StandardWriter, error) { gologger.Fatal().Msgf("Could not create output directory '%s': %s\n", options.StoreResponseDir, err) } } + writer := &StandardWriter{ - json: options.JSONL, - jsonReqResp: options.JSONRequests, - noMetadata: options.NoMeta, - matcherStatus: options.MatcherStatus, - timestamp: options.Timestamp, - aurora: auroraColorizer, - mutex: &sync.Mutex{}, - outputFile: outputFile, - traceFile: traceOutput, - errorFile: errorOutput, - severityColors: colorizer.New(auroraColorizer), - storeResponse: options.StoreResponse, - storeResponseDir: options.StoreResponseDir, + json: options.JSONL, + jsonReqResp: options.JSONRequests, + noMetadata: options.NoMeta, + matcherStatus: options.MatcherStatus, + timestamp: options.Timestamp, + aurora: auroraColorizer, + mutex: &sync.Mutex{}, + outputFile: outputFile, + traceFile: traceOutput, + errorFile: errorOutput, + severityColors: colorizer.New(auroraColorizer), + storeResponse: options.StoreResponse, + storeResponseDir: options.StoreResponseDir, } return writer, nil } diff --git a/v2/pkg/protocols/common/executer/executer.go b/v2/pkg/protocols/common/executer/executer.go index f9bae95979..8bfa1be0f4 100644 --- a/v2/pkg/protocols/common/executer/executer.go +++ b/v2/pkg/protocols/common/executer/executer.go @@ -70,6 +70,17 @@ func (e *Executer) Execute(input *contextargs.Context) (bool, error) { }) } previous := make(map[string]interface{}) + + var lastMatcherEvent *output.InternalWrappedEvent + writeFailureCallback := func(event *output.InternalWrappedEvent, matcherStatus bool) { + if !results.Load() && matcherStatus { + if err := e.options.Output.WriteFailure(event.InternalEvent); err != nil { + gologger.Warning().Msgf("Could not write failure event to output: %s\n", err) + } + results.CompareAndSwap(false, true) + } + } + for _, req := range e.requests { inputItem := input.Clone() if e.options.InputHelper != nil && input.MetaInput.Input != "" { @@ -94,16 +105,12 @@ func (e *Executer) Execute(input *contextargs.Context) (bool, error) { // in that case we can skip it, otherwise we've to show failure in // case of matcher-status flag. if !event.HasOperatorResult() && !event.UsesInteractsh { - if err := e.options.Output.WriteFailure(event.InternalEvent); err != nil { - gologger.Warning().Msgf("Could not write failure event to output: %s\n", err) - } + lastMatcherEvent = event } else { if writer.WriteResult(event, e.options.Output, e.options.Progress, e.options.IssuesClient) { results.CompareAndSwap(false, true) } else { - if err := e.options.Output.WriteFailure(event.InternalEvent); err != nil { - gologger.Warning().Msgf("Could not write failure event to output: %s\n", err) - } + lastMatcherEvent = event } } }) @@ -118,6 +125,9 @@ func (e *Executer) Execute(input *contextargs.Context) (bool, error) { break } } + if lastMatcherEvent != nil { + writeFailureCallback(lastMatcherEvent, e.options.Options.MatcherStatus) + } return results.Load(), nil } diff --git a/v2/pkg/protocols/network/request.go b/v2/pkg/protocols/network/request.go index d0f144a04e..5acb518d7e 100644 --- a/v2/pkg/protocols/network/request.go +++ b/v2/pkg/protocols/network/request.go @@ -61,6 +61,8 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata, actualAddress := replacer.Replace(kv.address, variables) if err := request.executeAddress(variables, actualAddress, address, input.MetaInput.Input, kv.tls, previous, callback); err != nil { + outputEvent := request.responseToDSLMap("", "", "", address, "") + callback(&output.InternalWrappedEvent{InternalEvent: outputEvent}) gologger.Warning().Msgf("[%v] Could not make network request for (%s) : %s\n", request.options.TemplateID, actualAddress, err) continue } diff --git a/v2/pkg/protocols/network/request_test.go b/v2/pkg/protocols/network/request_test.go index ff8b70134f..89c528724f 100644 --- a/v2/pkg/protocols/network/request_test.go +++ b/v2/pkg/protocols/network/request_test.go @@ -87,7 +87,7 @@ func TestNetworkExecuteWithResults(t *testing.T) { }) require.Nil(t, err, "could not execute network request") }) - require.Nil(t, finalEvent, "could not get event output from request") + require.Nil(t, finalEvent.Results, "could not get event output from request") request.Inputs[0].Type = NetworkInputTypeHolder{NetworkInputType: hexType} request.Inputs[0].Data = hex.EncodeToString([]byte(fmt.Sprintf("GET / HTTP/1.1\r\nHost: %s\r\n\r\n", parsed.Host))) diff --git a/v2/pkg/testutils/integration.go b/v2/pkg/testutils/integration.go index b3a90c5d0c..42777fd8f6 100644 --- a/v2/pkg/testutils/integration.go +++ b/v2/pkg/testutils/integration.go @@ -63,7 +63,7 @@ func RunNucleiBareArgsAndGetResults(debug bool, extra ...string) ([]string, erro if debug { fmt.Println(string(data)) } - if err != nil { + if len(data) < 1 && err != nil { return nil, fmt.Errorf("%v: %v", err.Error(), string(data)) } var parts []string