diff --git a/credsweeper/rules/config.yaml b/credsweeper/rules/config.yaml index 898ea587b..dbcfe72e8 100644 --- a/credsweeper/rules/config.yaml +++ b/credsweeper/rules/config.yaml @@ -131,7 +131,7 @@ confidence: strong type: pattern values: - - (?[0-9A-Fa-f]{8}(-[0-9A-Fa-f]{4}){3}-[0-9A-Fa-f]{12})(?![=0-9A-Za-z_+-]) + - (?[0-9A-Fa-f]{8}(-[0-9A-Fa-f]{4}){3}-[0-9A-Fa-f]{12})(?![0-9A-Za-z_-]) min_line_len: 36 required_substrings: - "-" @@ -148,7 +148,7 @@ confidence: moderate type: pattern values: - - (?(ABIA|ACCA|AGPA|AIDA|AIPA|AKIA|ANPA|ANVA|AROA|APKA|ASCA|ASIA)[0-9A-Z]{16,17})(?![=0-9A-Za-z_+-]) + - (?(ABIA|ACCA|AGPA|AIDA|AIPA|AKIA|ANPA|ANVA|AROA|APKA|ASCA|ASIA)[0-9A-Z]{16,17})(?![0-9A-Za-z_-]) filter_type: GeneralPattern required_substrings: - A @@ -163,8 +163,8 @@ confidence: moderate type: multi values: - - (?(ABIA|ACCA|AGPA|AIDA|AIPA|AKIA|ANPA|ANVA|AROA|APKA|ASCA|ASIA)[0-9A-Z]{16,17})(?![=0-9A-Za-z_+-]) - - (?[0-9A-Za-z/+]{35,80})(?![=0-9A-Za-z_/+-]) + - (?(ABIA|ACCA|AGPA|AIDA|AIPA|AKIA|ANPA|ANVA|AROA|APKA|ASCA|ASIA)[0-9A-Z]{16,17})(?![0-9A-Za-z_-]) + - (?[0-9A-Za-z/+]{35,80})(?![0-9A-Za-z_/+-]) filter_type: GeneralPattern required_substrings: - A @@ -179,7 +179,7 @@ confidence: strong type: pattern values: - - (?amzn\.mws\.[0-9a-z]{8}-[0-9a-z]{4}-[0-9a-z]{4}-[0-9a-z]{4}-[0-9a-z]{12})(?![=0-9A-Za-z_+-]) + - (?amzn\.mws\.[0-9a-z]{8}-[0-9a-z]{4}-[0-9a-z]{4}-[0-9a-z]{4}-[0-9a-z]{12})(?![0-9A-Za-z_-]) filter_type: GeneralPattern required_substrings: - amzn @@ -207,7 +207,7 @@ confidence: moderate type: pattern values: - - (?dt0[A-Za-z]{1}[0-9]{2}\.[0-9A-Z]{24}\.[0-9A-Z]{64})(?![=0-9A-Za-z_+-]) + - (?dt0[A-Za-z]{1}[0-9]{2}\.[0-9A-Z]{24}\.[0-9A-Z]{64})(?![0-9A-Za-z_-]) filter_type: GeneralPattern required_substrings: - dt0 @@ -221,7 +221,7 @@ confidence: moderate type: pattern values: - - (?EAA[0-9A-Za-z]{80,800}) + - (?EAA[0-9A-Za-z]{80,800}) filter_type: - ValuePatternCheck - ValueBase64PartCheck @@ -237,7 +237,7 @@ confidence: moderate type: pattern values: - - (?[0-9]{12,18}\|[0-9A-Za-z_-]{24,28})(?![=0-9A-Za-z_+-]) + - (?[0-9]{12,18}\|[0-9A-Za-z_-]{24,28})(?![0-9A-Za-z_-]) filter_type: GeneralPattern required_substrings: - "|" @@ -269,7 +269,7 @@ confidence: moderate type: pattern values: - - (?AIza[0-9A-Za-z_-]{35})(?![=0-9A-Za-z_+-]) + - (?AIza[0-9A-Za-z_-]{35})(?![0-9A-Za-z_-]) filter_type: GeneralPattern validations: - GoogleApiKeyValidation @@ -302,7 +302,7 @@ confidence: strong type: pattern values: - - (?GOCSPX-[0-9A-Za-z_-]{28})(?![=0-9A-Za-z_+-]) + - (?GOCSPX-[0-9A-Za-z_-]{28})(?![0-9A-Za-z_-]) filter_type: GeneralPattern required_substrings: - GOCSPX- @@ -316,7 +316,7 @@ confidence: moderate type: pattern values: - - (?ya29\.[0-9A-Za-z_-]{22,8000}) + - (?ya29\.[0-9A-Za-z_-]{22,8000}) filter_type: GeneralPattern required_substrings: - ya29. @@ -330,7 +330,7 @@ confidence: moderate type: pattern values: - - (?i)(?Pheroku(.{0,20})?[0-9a-f]{8}(-[0-9a-f]{4})+-[0-9a-f]{12})(?![=0-9A-Za-z_+-]) + - (?i)(?Pheroku(.{0,20})?[0-9a-f]{8}(-[0-9a-f]{4})+-[0-9a-f]{12})(?![0-9A-Za-z_-]) filter_type: GeneralPattern required_substrings: - heroku @@ -344,7 +344,7 @@ confidence: strong type: pattern values: - - (?IGQVJ[0-9A-Za-z_=-]{100,8000}) + - (?IGQVJ[0-9A-Za-z_=-]{100,8000}) filter_type: GeneralPattern required_substrings: - IGQVJ @@ -358,7 +358,7 @@ confidence: strong type: pattern values: - - (?eyJ[0-9A-Za-z_+/=-]{15,8000}(\.[0-9A-Za-z_+/=-]{0,8000}){2,16}) + - (?eyJ[0-9A-Za-z_+/=-]{15,8000}(\.[0-9A-Za-z_+/=-]{0,8000}){2,16}) filter_type: - ValueJsonWebTokenCheck required_substrings: @@ -373,7 +373,7 @@ confidence: moderate type: pattern values: - - (?[0-9A-Za-z_-]{32}-us[0-9]{1,2})(?![=0-9A-Za-z_+-]) + - (?[0-9A-Za-z_-]{32}-us[0-9]{1,2})(?![0-9A-Za-z_-]) filter_type: GeneralPattern validations: - MailChimpKeyValidation @@ -389,7 +389,7 @@ confidence: moderate type: pattern values: - - (?key-[0-9A-Za-z_-]{32})(?![=0-9A-Za-z_+-]) + - (?key-[0-9A-Za-z_-]{32})(?![0-9A-Za-z_-]) filter_type: GeneralPattern required_substrings: - key- @@ -418,7 +418,7 @@ confidence: strong type: pattern values: - - (?Paccess_token\$production\$[0-9a-z]{16}\$[0-9a-z]{32})(?![=0-9A-Za-z_+-]) + - (?Paccess_token\$production\$[0-9a-z]{16}\$[0-9a-z]{32})(?![0-9A-Za-z_-]) filter_type: GeneralPattern required_substrings: - access_token$production$ @@ -475,7 +475,7 @@ confidence: strong type: pattern values: - - (?Psk_live_[0-9a-z]{32})(?![=0-9A-Za-z_+-]) + - (?Psk_live_[0-9a-z]{32})(?![0-9A-Za-z_-]) filter_type: GeneralPattern required_substrings: - sk_live_ @@ -517,7 +517,7 @@ confidence: strong type: pattern values: - - (?Pshp(at|ca|pa|ss)_[0-9A-Fa-f]{32})(?![=0-9A-Za-z_+-]) + - (?Pshp(at|ca|pa|ss)_[0-9A-Fa-f]{32})(?![0-9A-Za-z_-]) filter_type: TokenPattern required_substrings: - shp @@ -531,7 +531,7 @@ confidence: strong type: pattern values: - - (?xox[aboprst]\-[0-9A-Za-z-]{10,250}) + - (?xox[aboprst]\-[0-9A-Za-z-]{10,250}) filter_type: GeneralPattern validations: - SlackTokenValidation @@ -561,7 +561,7 @@ confidence: strong type: pattern values: - - (?Psk_live_[0-9A-Za-z_-]{24})(?![=0-9A-Za-z_+-]) + - (?Psk_live_[0-9A-Za-z_-]{24})(?![0-9A-Za-z_-]) filter_type: GeneralPattern validations: - StripeApiKeyValidation @@ -577,7 +577,7 @@ confidence: strong type: pattern values: - - (?Prk_live_[0-9A-Za-z_-]{24})(?![=0-9A-Za-z_+-]) + - (?Prk_live_[0-9A-Za-z_-]{24})(?![0-9A-Za-z_-]) filter_type: GeneralPattern required_substrings: - rk_live_ @@ -591,7 +591,7 @@ confidence: moderate type: pattern values: - - (?EAAA[0-9A-Za-z_-]{60})(?![=0-9A-Za-z_+-]) + - (?EAAA[0-9A-Za-z_-]{60})(?![0-9A-Za-z_-]) filter_type: GeneralPattern validations: - SquareAccessTokenValidation @@ -607,7 +607,7 @@ confidence: strong type: pattern values: - - (?sq0[a-z]{3}-[0-9A-Za-z_-]{22})(?![=0-9A-Za-z_+-]) + - (?sq0[a-z]{3}-[0-9A-Za-z_-]{22})(?![0-9A-Za-z_-]) filter_type: GeneralPattern validations: - SquareClientIdValidation @@ -623,7 +623,7 @@ confidence: strong type: pattern values: - - (?Psq0csp-[0-9A-Za-z_-]{43})(?![=0-9A-Za-z_+-]) + - (?Psq0csp-[0-9A-Za-z_-]{43})(?![0-9A-Za-z_-]) filter_type: GeneralPattern required_substrings: - sq0csp @@ -651,7 +651,7 @@ confidence: moderate type: pattern values: - - (?SK[0-9A-Fa-f]{32})(?![=0-9A-Za-z_+-]) + - (?SK[0-9A-Fa-f]{32})(?![0-9A-Za-z_-]) filter_type: GeneralPattern required_substrings: - SK @@ -763,7 +763,7 @@ confidence: moderate type: pattern values: - - (?P[0-9]{8,10}:[0-9A-Za-z_-]{35})(?![=0-9A-Za-z_+-]) + - (?P[0-9]{8,10}:[0-9A-Za-z_-]{35})(?![0-9A-Za-z_-]) filter_type: GeneralPattern required_substrings: - :AA @@ -791,7 +791,7 @@ confidence: strong type: pattern values: - - (?gh[pousr]_[0-9A-Za-z_-]{36,255}) + - (?gh[pousr]_[0-9A-Za-z_-]{36,255}) filter_type: - ValueGitHubCheck validations: @@ -812,7 +812,7 @@ confidence: strong type: pattern values: - - (?github_pat_[0-9A-Za-z_]{80,255}) + - (?github_pat_[0-9A-Za-z_]{80,255}) filter_type: GeneralPattern validations: - GithubTokenValidation @@ -899,7 +899,7 @@ confidence: strong type: pattern values: - - (?(cmVmdGtuO[0-9A-Za-z_-]{55}|AKCp[0-9A-Za-z_-]{69}))(?![=0-9A-Za-z_+-]) + - (?(cmVmdGtuO[0-9A-Za-z_-]{55}|AKCp[0-9A-Za-z_-]{69}))(?![0-9A-Za-z_-]) filter_type: - ValueJfrogTokenCheck required_substrings: @@ -915,7 +915,7 @@ confidence: strong type: pattern values: - - (?eyJ[A-Za-z0-9_=-]{50,500}\.eyJ[A-Za-z0-9_=-]{8,8000}\.[A-Za-z0-9_=-]{18,800}) + - (?eyJ[0-9A-Za-z_=-]{50,500}\.eyJ[0-9A-Za-z_=-]{8,8000}\.[0-9A-Za-z_=-]{18,800}) filter_type: - ValueAzureTokenCheck required_substrings: @@ -930,7 +930,7 @@ confidence: moderate type: pattern values: - - (?[a-zA-Z0-9_~.-]{3}8Q~[a-zA-Z0-9_~.-]{34})(?![=0-9A-Za-z_+-]) + - (?[0-9A-Za-z_~.-]{3}8Q~[0-9A-Za-z_~.-]{34})(?![0-9A-Za-z_-]) filter_type: TokenPattern min_line_len: 40 required_substrings: @@ -944,7 +944,7 @@ confidence: strong type: pattern values: - - (?ATBB[A-Za-z0-9]{24}[A-F0-9]{8})(?![=0-9A-Za-z_+-]) + - (?ATBB[0-9A-Za-z]{24}[A-F0-9]{8})(?![0-9A-Za-z_-]) filter_type: - ValueAtlassianTokenCheck min_line_len: 28 @@ -959,7 +959,7 @@ confidence: strong type: pattern values: - - (?ATCTT3xFfGN0[a-zA-Z0-9_-]{80,800}(\\?=|%3[dD])[A-F0-9]{8})(?![=0-9A-Za-z_+-]) + - (?ATCTT3xFfGN0[0-9A-Za-z_-]{80,800}(\\?=|%3[dD])[A-F0-9]{8})(?![0-9A-Za-z_-]) filter_type: - ValueAtlassianTokenCheck min_line_len: 160 @@ -974,7 +974,7 @@ confidence: strong type: pattern values: - - (?BBDC-[NMO][ADgjQTwz][A-Za-z0-9_-]{42})(?![=0-9A-Za-z_+-]) + - (?BBDC-[NMO][ADgjQTwz][0-9A-Za-z_-]{42})(?![0-9A-Za-z_-]) filter_type: - ValueAtlassianTokenCheck min_line_len: 49 @@ -989,7 +989,7 @@ confidence: weak type: pattern values: - - (?[a-zA-Z0-9]{18}([a-zA-Z0-9]{14})?)(?![0-9A-Za-z.$_/+-]) + - (?[0-9A-Za-z]{18}([0-9A-Za-z]{14})?)(?![0-9A-Za-z.$_/+-]) filter_type: WeirdBase64Token min_line_len: 18 required_regex: "[0-9A-Za-z_/+-]{15}" @@ -1002,7 +1002,7 @@ confidence: weak type: pattern values: - - (?([a-zA-Z0-9_-]{32}){1,2})(?![0-9A-Za-z.$_/+-]) + - (?([0-9A-Za-z_-]{32}){1,2})(?![0-9A-Za-z.$_/+-]) filter_type: WeirdBase64Token min_line_len: 32 required_regex: "[0-9A-Za-z_/+-]{15}" @@ -1015,7 +1015,7 @@ confidence: strong type: pattern values: - - (?[NMO][ADgjQTwz][a-zA-Z0-9_-]{42})(?![=0-9A-Za-z_+-]) + - (?[NMO][ADgjQTwz][0-9A-Za-z_-]{42})(?![0-9A-Za-z_-]) filter_type: - ValueAtlassianTokenCheck min_line_len: 44 @@ -1033,7 +1033,7 @@ confidence: weak type: pattern values: - - (?[a-zA-Z0-9]{24})(?![=0-9A-Za-z.$_/+-]) + - (?[0-9A-Za-z]{24})(?![=0-9A-Za-z.$_/+-]) filter_type: WeirdBase64Token min_line_len: 24 required_regex: "[0-9A-Za-z_/+-]{15}" @@ -1046,7 +1046,7 @@ confidence: strong type: pattern values: - - (?ATATT3xFfGF0[a-zA-Z0-9_-]{80,800}(\\?=|%3[dD])[A-F0-9]{8})(?![=0-9A-Za-z_+-]) + - (?ATATT3xFfGF0[0-9A-Za-z_-]{80,800}(\\?=|%3[dD])[A-F0-9]{8})(?![0-9A-Za-z_-]) filter_type: - ValueAtlassianTokenCheck min_line_len: 160 @@ -1061,7 +1061,7 @@ confidence: strong type: pattern values: - - (?do[op]_v1_[a-f0-9]{64})(?![=0-9A-Za-z_+-]) + - (?do[op]_v1_[a-f0-9]{64})(?![0-9A-Za-z_-]) filter_type: TokenPattern min_line_len: 71 required_substrings: @@ -1076,7 +1076,7 @@ confidence: moderate type: pattern values: - - (?sl.[A-Za-z0-9_-]{135})(?![=0-9A-Za-z_+-]) + - (?sl.[0-9A-Za-z_-]{135})(?![0-9A-Za-z_-]) filter_type: TokenPattern min_line_len: 138 required_substrings: @@ -1090,7 +1090,7 @@ confidence: moderate type: pattern values: - - (?oy2[a-z0-9]{43})(?![=0-9A-Za-z_+-]) + - (?oy2[a-z0-9]{43})(?![0-9A-Za-z_-]) filter_type: TokenPattern min_line_len: 46 required_substrings: @@ -1104,7 +1104,7 @@ confidence: strong type: pattern values: - - (?glpat-[a-zA-Z0-9_-]{20})(?![=0-9A-Za-z_+-]) + - (?glpat-[0-9A-Za-z_-]{20})(?![0-9A-Za-z_-]) filter_type: TokenPattern min_line_len: 26 required_substrings: @@ -1118,7 +1118,7 @@ confidence: strong type: pattern values: - - (?glptt-[a-f0-9]{40})(?![=0-9A-Za-z_+-]) + - (?glptt-[a-f0-9]{40})(?![0-9A-Za-z_-]) filter_type: TokenPattern min_line_len: 46 required_substrings: @@ -1132,7 +1132,7 @@ confidence: strong type: pattern values: - - (?GR1348941[a-zA-Z0-9_-]{20})(?![=0-9A-Za-z_+-]) + - (?GR1348941[0-9A-Za-z_-]{20})(?![0-9A-Za-z_-]) filter_type: TokenPattern min_line_len: 29 required_substrings: @@ -1146,7 +1146,7 @@ confidence: strong type: pattern values: - - (?glrt-[a-zA-Z0-9_-]{20})(?![=0-9A-Za-z_+-]) + - (?glrt-[0-9A-Za-z_-]{20})(?![0-9A-Za-z_-]) filter_type: TokenPattern min_line_len: 25 required_substrings: @@ -1160,7 +1160,7 @@ confidence: strong type: pattern values: - - (?eyJ[=a-zA-Z0-9_-]{64,360})(?![=0-9A-Za-z_+-]) + - (?eyJ[=0-9A-Za-z_-]{64,360})(?![0-9A-Za-z_-]) filter_type: - ValueGrafanaCheck min_line_len: 67 @@ -1175,7 +1175,7 @@ confidence: strong type: pattern values: - - (?glc_eyJ[0-9A-Za-z_-]{80,360})(?![=0-9A-Za-z_+-]) + - (?glc_eyJ[0-9A-Za-z_-]{80,360})(?![0-9A-Za-z_-]) filter_type: - ValueGrafanaCheck min_line_len: 87 @@ -1190,7 +1190,7 @@ confidence: weak type: pattern values: - - (?[A-Za-z0-9]{10,12}[B-Za-z0-9]A{10,12}[B-Za-z0-9][A-Za-z0-9]{40,44})(?![=0-9A-Za-z_+-]) + - (?[0-9A-Za-z]{10,12}[B-Za-z0-9]A{10,12}[B-Za-z0-9][0-9A-Za-z]{40,44})(?![=0-9A-Za-z_-]) filter_type: [] min_line_len: 43 required_substrings: @@ -1243,7 +1243,7 @@ confidence: strong type: pattern values: - - (?[0-9A-Za-z_-]{14}\.atlasv1\.[0-9A-Za-z_-]{67})(?![=0-9A-Za-z_/+-]) + - (?[0-9A-Za-z_-]{14}\.atlasv1\.[0-9A-Za-z_-]{67})(?![0-9A-Za-z_-]) filter_type: - ValuePatternCheck min_line_len: 90 @@ -1276,7 +1276,7 @@ confidence: strong type: pattern values: - - (?sk-[0-9A-Za-z_-]{16,32}(T3BlbkFJ|9wZW5BS|PcGVuQU)[0-9A-Za-z_-]{16,32})(?![=0-9A-Za-z_/+-]) + - (?sk-[0-9A-Za-z_-]{16,32}(T3BlbkFJ|9wZW5BS|PcGVuQU)[0-9A-Za-z_-]{16,32})(?![0-9A-Za-z_-]) min_line_len: 51 filter_type: - ValuePatternCheck @@ -1294,7 +1294,7 @@ confidence: strong type: pattern values: - - (?SWMTKN-1-[0-9a-z]{50}-[0-9a-z]{25})(?![=0-9A-Za-z_/+-]) + - (?SWMTKN-1-[0-9a-z]{50}-[0-9a-z]{25})(?![0-9A-Za-z_-]) min_line_len: 85 filter_type: - ValueCoupleKeywordCheck @@ -1304,12 +1304,44 @@ - code - doc +- name: Groq API Key + severity: high + confidence: strong + type: pattern + values: + - (?gsk_[0-9A-Za-z_-]{52})(?![0-9A-Za-z_-]) + min_line_len: 56 + filter_type: + - ValuePatternCheck + - ValueEntropyBase64Check + required_substrings: + - gsk_ + target: + - code + - doc + +- name: Hugging Face User Access Token + severity: high + confidence: moderate + type: pattern + values: + - (?hf_[0-9A-Za-z_-]{34})(?![0-9A-Za-z_-]) + min_line_len: 37 + filter_type: + - ValuePatternCheck + - ValueEntropyBase64Check + required_substrings: + - hf_ + target: + - code + - doc + - name: Discord Bot Token severity: high confidence: strong type: pattern values: - - (?[NMO][ADgjQTwz][a-zA-Z0-9_-]{22,26}\.[a-zA-Z0-9_-]{6}\.[a-zA-Z0-9_-]{30,40})(?![0-9A-Za-z_/+-]) + - (?[NMO][ADgjQTwz][0-9A-Za-z_-]{22,26}\.[0-9A-Za-z_-]{6}\.[0-9A-Za-z_-]{30,40})(?![0-9A-Za-z_-]) min_line_len: 62 filter_type: - ValueDiscordBotCheck @@ -1327,7 +1359,7 @@ confidence: strong type: pattern values: - - (?glsa_[0-9A-Za-z_-]{32}_[0-9A-Fa-f]{8})(?![=0-9A-Za-z_+-]) + - (?glsa_[0-9A-Za-z_-]{32}_[0-9A-Fa-f]{8})(?![0-9A-Za-z_-]) min_line_len: 46 filter_type: - ValueGrafanaServiceCheck diff --git a/tests/__init__.py b/tests/__init__.py index 880b090ea..575514bac 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1,20 +1,20 @@ from pathlib import Path # total number of files in test samples -SAMPLES_FILES_COUNT: int = 134 +SAMPLES_FILES_COUNT: int = 136 # the lowest value of ML threshold is used to display possible lowest values NEGLIGIBLE_ML_THRESHOLD = 0.0001 # credentials count after scan -SAMPLES_CRED_COUNT: int = 391 -SAMPLES_CRED_LINE_COUNT: int = 409 +SAMPLES_CRED_COUNT: int = 396 +SAMPLES_CRED_LINE_COUNT: int = 414 # credentials count after post-processing -SAMPLES_POST_CRED_COUNT: int = 344 +SAMPLES_POST_CRED_COUNT: int = 349 # with option --doc -SAMPLES_IN_DOC = 425 +SAMPLES_IN_DOC = 430 # archived credentials that are not found without --depth SAMPLES_IN_DEEP_1 = SAMPLES_POST_CRED_COUNT + 23 diff --git a/tests/data/depth_3.json b/tests/data/depth_3.json index dfe52c9b2..14bd8ae9f 100644 --- a/tests/data/depth_3.json +++ b/tests/data/depth_3.json @@ -7226,6 +7226,60 @@ } ] }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, + "rule": "Groq API Key", + "severity": "high", + "confidence": "strong", + "line_data_list": [ + { + "line": "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", + "line_num": 1, + "path": "./tests/samples/groq_api_key", + "info": "./tests/samples/groq_api_key|RAW", + "value": "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", + "value_start": 0, + "value_end": 56, + "variable": null, + "variable_start": -2, + "variable_end": -2, + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 5.096509298449433, + "valid": true + } + } + ] + }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, + "rule": "Groq API Key", + "severity": "high", + "confidence": "strong", + "line_data_list": [ + { + "line": "url/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?part", + "line_num": 2, + "path": "./tests/samples/groq_api_key", + "info": "./tests/samples/groq_api_key|RAW", + "value": "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", + "value_start": 4, + "value_end": 60, + "variable": null, + "variable_start": -2, + "variable_end": -2, + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 5.096509298449433, + "valid": true + } + } + ] + }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", @@ -7280,6 +7334,87 @@ } ] }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, + "rule": "Hugging Face User Access Token", + "severity": "high", + "confidence": "moderate", + "line_data_list": [ + { + "line": "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", + "line_num": 1, + "path": "./tests/samples/huggingface", + "info": "./tests/samples/huggingface|RAW", + "value": "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", + "value_start": 0, + "value_end": 37, + "variable": null, + "variable_start": -2, + "variable_end": -2, + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 4.5617684773102365, + "valid": true + } + } + ] + }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, + "rule": "Hugging Face User Access Token", + "severity": "high", + "confidence": "moderate", + "line_data_list": [ + { + "line": "url/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/part", + "line_num": 2, + "path": "./tests/samples/huggingface", + "info": "./tests/samples/huggingface|RAW", + "value": "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", + "value_start": 4, + "value_end": 41, + "variable": null, + "variable_start": -2, + "variable_end": -2, + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 4.636224896287629, + "valid": true + } + } + ] + }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, + "rule": "Hugging Face User Access Token", + "severity": "high", + "confidence": "moderate", + "line_data_list": [ + { + "line": "hf_UdYjVqYvybBLEhIrwwEUYjOgkeyexample", + "line_num": 6, + "path": "./tests/samples/huggingface", + "info": "./tests/samples/huggingface|RAW", + "value": "hf_UdYjVqYvybBLEhIrwwEUYjOgkeyexample", + "value_start": 0, + "value_end": 37, + "variable": null, + "variable_start": -2, + "variable_end": -2, + "entropy_validation": { + "iterator": "BASE36_CHARS", + "entropy": 3.034378196565712, + "valid": true + } + } + ] + }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", diff --git a/tests/data/doc.json b/tests/data/doc.json index e99168514..cfb9d695a 100644 --- a/tests/data/doc.json +++ b/tests/data/doc.json @@ -11571,6 +11571,60 @@ } ] }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, + "rule": "Groq API Key", + "severity": "high", + "confidence": "strong", + "line_data_list": [ + { + "line": "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", + "line_num": 1, + "path": "./tests/samples/groq_api_key", + "info": "./tests/samples/groq_api_key|RAW", + "value": "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", + "value_start": 0, + "value_end": 56, + "variable": null, + "variable_start": -2, + "variable_end": -2, + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 5.096509298449433, + "valid": true + } + } + ] + }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, + "rule": "Groq API Key", + "severity": "high", + "confidence": "strong", + "line_data_list": [ + { + "line": "url/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?part", + "line_num": 2, + "path": "./tests/samples/groq_api_key", + "info": "./tests/samples/groq_api_key|RAW", + "value": "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", + "value_start": 4, + "value_end": 60, + "variable": null, + "variable_start": -2, + "variable_end": -2, + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 5.096509298449433, + "valid": true + } + } + ] + }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", @@ -11625,6 +11679,87 @@ } ] }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, + "rule": "Hugging Face User Access Token", + "severity": "high", + "confidence": "moderate", + "line_data_list": [ + { + "line": "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", + "line_num": 1, + "path": "./tests/samples/huggingface", + "info": "./tests/samples/huggingface|RAW", + "value": "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", + "value_start": 0, + "value_end": 37, + "variable": null, + "variable_start": -2, + "variable_end": -2, + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 4.5617684773102365, + "valid": true + } + } + ] + }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, + "rule": "Hugging Face User Access Token", + "severity": "high", + "confidence": "moderate", + "line_data_list": [ + { + "line": "url/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/part", + "line_num": 2, + "path": "./tests/samples/huggingface", + "info": "./tests/samples/huggingface|RAW", + "value": "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", + "value_start": 4, + "value_end": 41, + "variable": null, + "variable_start": -2, + "variable_end": -2, + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 4.636224896287629, + "valid": true + } + } + ] + }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, + "rule": "Hugging Face User Access Token", + "severity": "high", + "confidence": "moderate", + "line_data_list": [ + { + "line": "hf_UdYjVqYvybBLEhIrwwEUYjOgkeyexample", + "line_num": 6, + "path": "./tests/samples/huggingface", + "info": "./tests/samples/huggingface|RAW", + "value": "hf_UdYjVqYvybBLEhIrwwEUYjOgkeyexample", + "value_start": 0, + "value_end": 37, + "variable": null, + "variable_start": -2, + "variable_end": -2, + "entropy_validation": { + "iterator": "BASE36_CHARS", + "entropy": 3.034378196565712, + "valid": true + } + } + ] + }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", diff --git a/tests/data/ml_threshold.json b/tests/data/ml_threshold.json index f0e16516e..538d3716f 100644 --- a/tests/data/ml_threshold.json +++ b/tests/data/ml_threshold.json @@ -8130,6 +8130,60 @@ } ] }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, + "rule": "Groq API Key", + "severity": "high", + "confidence": "strong", + "line_data_list": [ + { + "line": "302892d5d7ff20d2939e6ae830464ff94534d7c8aba20300273fd1f8801621d4", + "line_num": 1, + "path": "./tests/samples/groq_api_key", + "info": "", + "value": "302892d5d7ff20d2939e6ae830464ff94534d7c8aba20300273fd1f8801621d4", + "value_start": 0, + "value_end": 56, + "variable": null, + "variable_start": -2, + "variable_end": -2, + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 5.096509298449433, + "valid": true + } + } + ] + }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, + "rule": "Groq API Key", + "severity": "high", + "confidence": "strong", + "line_data_list": [ + { + "line": "71d8ac3b132fee7e37c34f4777f5180033feab29bf8d88908c75498d76f0bdc7", + "line_num": 2, + "path": "./tests/samples/groq_api_key", + "info": "", + "value": "302892d5d7ff20d2939e6ae830464ff94534d7c8aba20300273fd1f8801621d4", + "value_start": 4, + "value_end": 60, + "variable": null, + "variable_start": -2, + "variable_end": -2, + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 5.096509298449433, + "valid": true + } + } + ] + }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", @@ -8184,6 +8238,87 @@ } ] }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, + "rule": "Hugging Face User Access Token", + "severity": "high", + "confidence": "moderate", + "line_data_list": [ + { + "line": "c7c5f3010331a178db9d2eba545072f051995fb3573bee5a06dfd6f09230495a", + "line_num": 1, + "path": "./tests/samples/huggingface", + "info": "", + "value": "c7c5f3010331a178db9d2eba545072f051995fb3573bee5a06dfd6f09230495a", + "value_start": 0, + "value_end": 37, + "variable": null, + "variable_start": -2, + "variable_end": -2, + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 4.5617684773102365, + "valid": true + } + } + ] + }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, + "rule": "Hugging Face User Access Token", + "severity": "high", + "confidence": "moderate", + "line_data_list": [ + { + "line": "12735376cb8a61fad65d0b390fa604867d2f7350af1a083b9818c0ecc94f3015", + "line_num": 2, + "path": "./tests/samples/huggingface", + "info": "", + "value": "d7a84b1a69cdca4fd8475e93d3cc78e95d11b51ffadb0587e4eb6f2373bfd211", + "value_start": 4, + "value_end": 41, + "variable": null, + "variable_start": -2, + "variable_end": -2, + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 4.636224896287629, + "valid": true + } + } + ] + }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, + "rule": "Hugging Face User Access Token", + "severity": "high", + "confidence": "moderate", + "line_data_list": [ + { + "line": "b4dd4b570dc14107d39ba82774bae722e9ccd1b8a279c0062a41b6431026c4ea", + "line_num": 6, + "path": "./tests/samples/huggingface", + "info": "", + "value": "b4dd4b570dc14107d39ba82774bae722e9ccd1b8a279c0062a41b6431026c4ea", + "value_start": 0, + "value_end": 37, + "variable": null, + "variable_start": -2, + "variable_end": -2, + "entropy_validation": { + "iterator": "BASE36_CHARS", + "entropy": 3.034378196565712, + "valid": true + } + } + ] + }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", diff --git a/tests/data/output.json b/tests/data/output.json index 7be36be16..8ec66fd3a 100644 --- a/tests/data/output.json +++ b/tests/data/output.json @@ -6969,6 +6969,60 @@ } ] }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, + "rule": "Groq API Key", + "severity": "high", + "confidence": "strong", + "line_data_list": [ + { + "line": "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", + "line_num": 1, + "path": "./tests/samples/groq_api_key", + "info": "", + "value": "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", + "value_start": 0, + "value_end": 56, + "variable": null, + "variable_start": -2, + "variable_end": -2, + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 5.096509298449433, + "valid": true + } + } + ] + }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, + "rule": "Groq API Key", + "severity": "high", + "confidence": "strong", + "line_data_list": [ + { + "line": "url/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?part", + "line_num": 2, + "path": "./tests/samples/groq_api_key", + "info": "", + "value": "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", + "value_start": 4, + "value_end": 60, + "variable": null, + "variable_start": -2, + "variable_end": -2, + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 5.096509298449433, + "valid": true + } + } + ] + }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", @@ -7023,6 +7077,87 @@ } ] }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, + "rule": "Hugging Face User Access Token", + "severity": "high", + "confidence": "moderate", + "line_data_list": [ + { + "line": "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", + "line_num": 1, + "path": "./tests/samples/huggingface", + "info": "", + "value": "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", + "value_start": 0, + "value_end": 37, + "variable": null, + "variable_start": -2, + "variable_end": -2, + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 4.5617684773102365, + "valid": true + } + } + ] + }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, + "rule": "Hugging Face User Access Token", + "severity": "high", + "confidence": "moderate", + "line_data_list": [ + { + "line": "url/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/part", + "line_num": 2, + "path": "./tests/samples/huggingface", + "info": "", + "value": "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", + "value_start": 4, + "value_end": 41, + "variable": null, + "variable_start": -2, + "variable_end": -2, + "entropy_validation": { + "iterator": "BASE64_CHARS", + "entropy": 4.636224896287629, + "valid": true + } + } + ] + }, + { + "api_validation": "NOT_AVAILABLE", + "ml_validation": "NOT_AVAILABLE", + "ml_probability": null, + "rule": "Hugging Face User Access Token", + "severity": "high", + "confidence": "moderate", + "line_data_list": [ + { + "line": "hf_UdYjVqYvybBLEhIrwwEUYjOgkeyexample", + "line_num": 6, + "path": "./tests/samples/huggingface", + "info": "", + "value": "hf_UdYjVqYvybBLEhIrwwEUYjOgkeyexample", + "value_start": 0, + "value_end": 37, + "variable": null, + "variable_start": -2, + "variable_end": -2, + "entropy_validation": { + "iterator": "BASE36_CHARS", + "entropy": 3.034378196565712, + "valid": true + } + } + ] + }, { "api_validation": "NOT_AVAILABLE", "ml_validation": "NOT_AVAILABLE", diff --git a/tests/samples/groq_api_key b/tests/samples/groq_api_key new file mode 100644 index 000000000..407d6b35f --- /dev/null +++ b/tests/samples/groq_api_key @@ -0,0 +1,2 @@ +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 +url/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?part \ No newline at end of file diff --git a/tests/samples/huggingface b/tests/samples/huggingface new file mode 100644 index 000000000..8d8aef6fc --- /dev/null +++ b/tests/samples/huggingface @@ -0,0 +1,6 @@ +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 +url/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/part + +# negative cases +hf_3478658437658347638475687346587374 +hf_UdYjVqYvybBLEhIrwwEUYjOgkeyexample