.github
1 year ago
.gitignore
1 year ago
.travis.yml
2 years ago
LICENSE
3 years ago
MANIFEST.in
2 years ago
README.md
1 year ago
__init__.py
1 year ago
composer.json
3 years ago
crawler-user-agents.json
1 year ago
format.js
1 year ago
go.mod
2 years ago
go.sum
2 years ago
index.d.ts
3 years ago
main.php
3 years ago
package-lock.json
2 years ago
package.json
2 years ago
pyproject.toml
1 year ago
test_harness.py
1 year ago
test_validation.py
3 years ago
validate.go
1 year ago
validate.php
3 years ago
validate.py
2 years ago
validate_test.go
1 year ago
validate_test.go
580 lines
| 1 | package agents |
| 2 | |
| 3 | import ( |
| 4 | "encoding/json" |
| 5 | "fmt" |
| 6 | "net/http" |
| 7 | "reflect" |
| 8 | "regexp/syntax" |
| 9 | "sort" |
| 10 | "strings" |
| 11 | "testing" |
| 12 | ) |
| 13 | |
| 14 | // TestAnalyzePattern tests analyzePattern function on many cases, including |
| 15 | // edge cases. |
| 16 | func TestAnalyzePattern(t *testing.T) { |
| 17 | cases := []struct { |
| 18 | input string |
| 19 | wantError string |
| 20 | wantPatterns []string |
| 21 | wantRe bool |
| 22 | shouldMatchRe []string |
| 23 | shouldNotMatchRe []string |
| 24 | }{ |
| 25 | { |
| 26 | input: "simple phrase", |
| 27 | wantPatterns: []string{"simple phrase"}, |
| 28 | }, |
| 29 | { |
| 30 | input: "^begin anchor", |
| 31 | wantPatterns: []string{"^begin anchor"}, |
| 32 | }, |
| 33 | { |
| 34 | input: "end anchor$", |
| 35 | wantPatterns: []string{"end anchor$"}, |
| 36 | }, |
| 37 | { |
| 38 | input: "^both anchors$", |
| 39 | wantPatterns: []string{"^both anchors$"}, |
| 40 | }, |
| 41 | { |
| 42 | input: "(alter|nation)", |
| 43 | wantPatterns: []string{"alter", "nation"}, |
| 44 | }, |
| 45 | { |
| 46 | input: "too many [aA][lL][tT][eE][rR][nN][aA][tT][iI][oO][nN][sS]", |
| 47 | wantPatterns: []string{"too many "}, |
| 48 | wantRe: true, |
| 49 | shouldMatchRe: []string{"too many ALTERNATIONs"}, |
| 50 | shouldNotMatchRe: []string{"too many combinations "}, |
| 51 | }, |
| 52 | { |
| 53 | input: "(alter|nation) concatenation (alter|nation)", |
| 54 | wantPatterns: []string{ |
| 55 | "alter concatenation alter", |
| 56 | "alter concatenation nation", |
| 57 | "nation concatenation alter", |
| 58 | "nation concatenation nation", |
| 59 | }, |
| 60 | }, |
| 61 | { |
| 62 | input: "clas[sS] of [c]haract[eiu]rs", |
| 63 | wantPatterns: []string{ |
| 64 | "clasS of characters", |
| 65 | "clasS of charactirs", |
| 66 | "clasS of characturs", |
| 67 | "class of characters", |
| 68 | "class of charactirs", |
| 69 | "class of characturs", |
| 70 | }, |
| 71 | }, |
| 72 | { |
| 73 | input: "ranges [0-3]x[a-c]", |
| 74 | wantPatterns: []string{ |
| 75 | "ranges 0xa", "ranges 0xb", "ranges 0xc", |
| 76 | "ranges 1xa", "ranges 1xb", "ranges 1xc", |
| 77 | "ranges 2xa", "ranges 2xb", "ranges 2xc", |
| 78 | "ranges 3xa", "ranges 3xb", "ranges 3xc", |
| 79 | }, |
| 80 | }, |
| 81 | { |
| 82 | input: "Quest?", |
| 83 | wantPatterns: []string{"Ques", "Quest"}, |
| 84 | }, |
| 85 | { |
| 86 | input: "Q?ue(st)?", |
| 87 | wantPatterns: []string{"Que", "Quest", "ue", "uest"}, |
| 88 | }, |
| 89 | { |
| 90 | input: "too many combinations [0-9][a-z]", |
| 91 | wantPatterns: []string{"too many combinations "}, |
| 92 | wantRe: true, |
| 93 | shouldMatchRe: []string{"too many combinations 0a"}, |
| 94 | shouldNotMatchRe: []string{"too many combinations "}, |
| 95 | }, |
| 96 | { |
| 97 | input: "negation in char class [^x]", |
| 98 | wantPatterns: []string{"negation in char class "}, |
| 99 | wantRe: true, |
| 100 | shouldMatchRe: []string{"negation in char class y"}, |
| 101 | shouldNotMatchRe: []string{"negation in char class x"}, |
| 102 | }, |
| 103 | { |
| 104 | input: "any char .", |
| 105 | wantPatterns: []string{"any char "}, |
| 106 | wantRe: true, |
| 107 | shouldMatchRe: []string{"any char x"}, |
| 108 | shouldNotMatchRe: []string{"any char_x"}, |
| 109 | }, |
| 110 | { |
| 111 | input: `word \boundary`, |
| 112 | wantPatterns: []string{"oundary"}, |
| 113 | wantRe: true, |
| 114 | shouldMatchRe: []string{"word oundary"}, |
| 115 | shouldNotMatchRe: []string{"word boundary"}, |
| 116 | }, |
| 117 | { |
| 118 | input: "asterisk*", |
| 119 | wantPatterns: []string{"asteris"}, |
| 120 | wantRe: true, |
| 121 | shouldMatchRe: []string{"asteris", "asterisk", "asteriskk"}, |
| 122 | shouldNotMatchRe: []string{"asterik"}, |
| 123 | }, |
| 124 | { |
| 125 | input: "plus+", |
| 126 | wantPatterns: []string{"plu"}, |
| 127 | wantRe: true, |
| 128 | shouldMatchRe: []string{"plus", "pluss"}, |
| 129 | shouldNotMatchRe: []string{"plu"}, |
| 130 | }, |
| 131 | { |
| 132 | input: "repeat{3,5}$", |
| 133 | wantPatterns: []string{"repeattt$", "repeatttt$", "repeattttt$"}, |
| 134 | }, |
| 135 | { |
| 136 | input: "repeat{1,120}$", |
| 137 | wantPatterns: []string{"repea"}, |
| 138 | wantRe: true, |
| 139 | shouldMatchRe: []string{"repeattt", "repeatttt", "repeattttt"}, |
| 140 | shouldNotMatchRe: []string{"repea5"}, |
| 141 | }, |
| 142 | { |
| 143 | input: "broken re[", |
| 144 | wantError: "does not compile", |
| 145 | }, |
| 146 | { |
| 147 | input: "n?o? ?l?o?n?g? ?l?i?t?e?r?a?l?", |
| 148 | wantError: "does not contain sufficiently long literal", |
| 149 | }, |
| 150 | } |
| 151 | |
| 152 | for _, tc := range cases { |
| 153 | tc := tc |
| 154 | |
| 155 | t.Run(tc.input, func(t *testing.T) { |
| 156 | gotPatterns, re, err := analyzePattern(tc.input) |
| 157 | if tc.wantError != "" { |
| 158 | if err == nil { |
| 159 | t.Fatalf("expected to get an error, got success") |
| 160 | } |
| 161 | if !strings.Contains(err.Error(), tc.wantError) { |
| 162 | t.Fatalf("the error returned must contain text %q, got %q", tc.wantError, err.Error()) |
| 163 | } |
| 164 | |
| 165 | return |
| 166 | } |
| 167 | |
| 168 | if err != nil { |
| 169 | t.Fatalf("unexpected error: %v", err) |
| 170 | } |
| 171 | |
| 172 | sort.Strings(tc.wantPatterns) |
| 173 | sort.Strings(gotPatterns) |
| 174 | if !reflect.DeepEqual(tc.wantPatterns, gotPatterns) { |
| 175 | t.Fatalf("returned list of patterns (%#v) does not match the expected value (%#v)", gotPatterns, tc.wantPatterns) |
| 176 | } |
| 177 | |
| 178 | if !tc.wantRe { |
| 179 | if re != nil { |
| 180 | t.Fatalf("unexpectedly got a re") |
| 181 | } |
| 182 | |
| 183 | return |
| 184 | } |
| 185 | |
| 186 | if re == nil { |
| 187 | t.Fatalf("expected to get a re, got nil") |
| 188 | } |
| 189 | for _, text := range tc.shouldMatchRe { |
| 190 | if !re.MatchString(text) { |
| 191 | t.Fatalf("test %q must match against the re, but it doesn't", text) |
| 192 | } |
| 193 | } |
| 194 | for _, text := range tc.shouldNotMatchRe { |
| 195 | if re.MatchString(text) { |
| 196 | t.Fatalf("test %q must not match against the re, but it does", text) |
| 197 | } |
| 198 | } |
| 199 | }) |
| 200 | } |
| 201 | } |
| 202 | |
| 203 | // TestLiteralizeRegexp tests expansion of a regexp to a list of literals. |
| 204 | func TestLiteralizeRegexp(t *testing.T) { |
| 205 | cases := []struct { |
| 206 | input string |
| 207 | maxLiterals int |
| 208 | wantOutput []string |
| 209 | wantOverflow bool |
| 210 | }{ |
| 211 | { |
| 212 | input: "simple phrase", |
| 213 | maxLiterals: 100, |
| 214 | wantOutput: []string{"simple phrase"}, |
| 215 | }, |
| 216 | { |
| 217 | input: "cases [1-2x-z]", |
| 218 | maxLiterals: 100, |
| 219 | wantOutput: []string{"cases 1", "cases 2", "cases x", "cases y", "cases z"}, |
| 220 | }, |
| 221 | { |
| 222 | input: "[Ii]gnore case", |
| 223 | maxLiterals: 100, |
| 224 | wantOutput: []string{"Ignore case", "ignore case"}, |
| 225 | }, |
| 226 | { |
| 227 | input: "overflow [1-2x-z]", |
| 228 | maxLiterals: 2, |
| 229 | wantOverflow: true, |
| 230 | }, |
| 231 | } |
| 232 | |
| 233 | for _, tc := range cases { |
| 234 | tc := tc |
| 235 | |
| 236 | t.Run(tc.input, func(t *testing.T) { |
| 237 | re, err := syntax.Parse(tc.input, syntax.Perl) |
| 238 | if err != nil { |
| 239 | t.Fatalf("failed to parse regexp %q: %v", tc.input, err) |
| 240 | } |
| 241 | |
| 242 | gotPatterns, ok := literalizeRegexp(re, tc.maxLiterals) |
| 243 | if tc.wantOverflow { |
| 244 | if ok { |
| 245 | t.Fatalf("expected to get an overflow, got success") |
| 246 | } |
| 247 | |
| 248 | return |
| 249 | } |
| 250 | |
| 251 | if !ok { |
| 252 | t.Fatalf("unexpected overflow") |
| 253 | } |
| 254 | |
| 255 | sort.Strings(tc.wantOutput) |
| 256 | sort.Strings(gotPatterns) |
| 257 | if !reflect.DeepEqual(tc.wantOutput, gotPatterns) { |
| 258 | t.Fatalf("returned list of patterns (%#v) does not match the expected value (%#v)", gotPatterns, tc.wantOutput) |
| 259 | } |
| 260 | }) |
| 261 | } |
| 262 | } |
| 263 | |
| 264 | // TestCombinations tests combinations() function. |
| 265 | func TestCombinations(t *testing.T) { |
| 266 | cases := []struct { |
| 267 | name string |
| 268 | input [][]string |
| 269 | maxLiterals int |
| 270 | wantOutput []string |
| 271 | wantOverflow bool |
| 272 | }{ |
| 273 | { |
| 274 | name: "1x1", |
| 275 | input: [][]string{{"A"}, {"B"}}, |
| 276 | maxLiterals: 100, |
| 277 | wantOutput: []string{"AB"}, |
| 278 | }, |
| 279 | { |
| 280 | name: "0x1", |
| 281 | input: [][]string{{}, {"B"}}, |
| 282 | maxLiterals: 100, |
| 283 | wantOutput: []string{}, |
| 284 | }, |
| 285 | { |
| 286 | name: "1x2", |
| 287 | input: [][]string{{"A"}, {"1", "2"}}, |
| 288 | maxLiterals: 100, |
| 289 | wantOutput: []string{"A1", "A2"}, |
| 290 | }, |
| 291 | { |
| 292 | name: "2x2", |
| 293 | input: [][]string{{"A", "B"}, {"1", "2"}}, |
| 294 | maxLiterals: 100, |
| 295 | wantOutput: []string{"A1", "A2", "B1", "B2"}, |
| 296 | }, |
| 297 | { |
| 298 | name: "empty string as an option", |
| 299 | input: [][]string{{"A", ""}, {"1", "2"}}, |
| 300 | maxLiterals: 100, |
| 301 | wantOutput: []string{"A1", "A2", "1", "2"}, |
| 302 | }, |
| 303 | { |
| 304 | name: "overflow", |
| 305 | input: [][]string{{"A", "B"}, {"1", "2"}}, |
| 306 | maxLiterals: 3, |
| 307 | wantOverflow: true, |
| 308 | }, |
| 309 | } |
| 310 | |
| 311 | for _, tc := range cases { |
| 312 | tc := tc |
| 313 | |
| 314 | t.Run(tc.name, func(t *testing.T) { |
| 315 | gotPatterns, ok := combinations(tc.input, tc.maxLiterals) |
| 316 | if tc.wantOverflow { |
| 317 | if ok { |
| 318 | t.Fatalf("expected to get an overflow, got success") |
| 319 | } |
| 320 | |
| 321 | return |
| 322 | } |
| 323 | |
| 324 | if !ok { |
| 325 | t.Fatalf("unexpected overflow") |
| 326 | } |
| 327 | |
| 328 | sort.Strings(tc.wantOutput) |
| 329 | sort.Strings(gotPatterns) |
| 330 | if !reflect.DeepEqual(tc.wantOutput, gotPatterns) { |
| 331 | t.Fatalf("returned list of patterns (%#v) does not match the expected value (%#v)", gotPatterns, tc.wantOutput) |
| 332 | } |
| 333 | }) |
| 334 | } |
| 335 | } |
| 336 | |
| 337 | // TestUnwrapCase tests unwrapping literals of case-insensitive regexps. |
| 338 | func TestUnwrapCase(t *testing.T) { |
| 339 | cases := []struct { |
| 340 | name string |
| 341 | ignoreCase bool |
| 342 | inputPatterns []string |
| 343 | maxLiterals int |
| 344 | wantOutput []string |
| 345 | wantOverflow bool |
| 346 | }{ |
| 347 | { |
| 348 | name: "simple phrase", |
| 349 | inputPatterns: []string{"simple phrase"}, |
| 350 | maxLiterals: 100, |
| 351 | wantOutput: []string{"simple phrase"}, |
| 352 | }, |
| 353 | { |
| 354 | name: "ignore case", |
| 355 | ignoreCase: true, |
| 356 | inputPatterns: []string{"i"}, |
| 357 | maxLiterals: 100, |
| 358 | wantOutput: []string{"i", "I"}, |
| 359 | }, |
| 360 | { |
| 361 | name: "ignore case two letters", |
| 362 | ignoreCase: true, |
| 363 | inputPatterns: []string{"ic"}, |
| 364 | maxLiterals: 100, |
| 365 | wantOutput: []string{"IC", "Ic", "iC", "ic"}, |
| 366 | }, |
| 367 | { |
| 368 | name: "ignore case two words", |
| 369 | ignoreCase: true, |
| 370 | inputPatterns: []string{"i", "c"}, |
| 371 | maxLiterals: 100, |
| 372 | wantOutput: []string{"C", "I", "c", "i"}, |
| 373 | }, |
| 374 | { |
| 375 | name: "ignore case overflow", |
| 376 | ignoreCase: true, |
| 377 | inputPatterns: []string{"long text"}, |
| 378 | maxLiterals: 100, |
| 379 | wantOverflow: true, |
| 380 | }, |
| 381 | } |
| 382 | |
| 383 | for _, tc := range cases { |
| 384 | tc := tc |
| 385 | |
| 386 | t.Run(tc.name, func(t *testing.T) { |
| 387 | re := &syntax.Regexp{} |
| 388 | if tc.ignoreCase { |
| 389 | re.Flags = syntax.FoldCase |
| 390 | } |
| 391 | |
| 392 | gotPatterns, ok := unwrapCase(re, tc.inputPatterns, tc.maxLiterals) |
| 393 | if tc.wantOverflow { |
| 394 | if ok { |
| 395 | t.Fatalf("expected to get an overflow, got success") |
| 396 | } |
| 397 | |
| 398 | return |
| 399 | } |
| 400 | |
| 401 | if !ok { |
| 402 | t.Fatalf("unexpected overflow") |
| 403 | } |
| 404 | |
| 405 | sort.Strings(tc.wantOutput) |
| 406 | sort.Strings(gotPatterns) |
| 407 | if !reflect.DeepEqual(tc.wantOutput, gotPatterns) { |
| 408 | t.Fatalf("returned list of patterns (%#v) does not match the expected value (%#v)", gotPatterns, tc.wantOutput) |
| 409 | } |
| 410 | }) |
| 411 | } |
| 412 | } |
| 413 | |
| 414 | // TestFindLongestCommonLiteral tests finding longest literal in a regexp. |
| 415 | func TestFindLongestCommonLiteral(t *testing.T) { |
| 416 | cases := []struct { |
| 417 | input string |
| 418 | wantOutput string |
| 419 | }{ |
| 420 | { |
| 421 | input: "simple phrase", |
| 422 | wantOutput: "simple phrase", |
| 423 | }, |
| 424 | { |
| 425 | input: "simple (phrase)?", |
| 426 | wantOutput: "simple ", |
| 427 | }, |
| 428 | { |
| 429 | input: "[iI]", |
| 430 | wantOutput: "", |
| 431 | }, |
| 432 | { |
| 433 | input: "[i]b", |
| 434 | wantOutput: "ib", |
| 435 | }, |
| 436 | { |
| 437 | input: "simple (phrase)+", |
| 438 | wantOutput: "simple ", |
| 439 | }, |
| 440 | { |
| 441 | input: "a*", |
| 442 | wantOutput: "", |
| 443 | }, |
| 444 | { |
| 445 | input: "(abc)|(ab)", |
| 446 | wantOutput: "", |
| 447 | }, |
| 448 | } |
| 449 | |
| 450 | for _, tc := range cases { |
| 451 | tc := tc |
| 452 | |
| 453 | t.Run(tc.input, func(t *testing.T) { |
| 454 | re, err := syntax.Parse(tc.input, syntax.Perl) |
| 455 | if err != nil { |
| 456 | t.Fatalf("failed to parse regexp %q: %v", tc.input, err) |
| 457 | } |
| 458 | |
| 459 | gotOutput := findLongestCommonLiteral(re) |
| 460 | |
| 461 | if gotOutput != tc.wantOutput { |
| 462 | t.Fatalf("returned value (%q) does not match the expected value (%q)", gotOutput, tc.wantOutput) |
| 463 | } |
| 464 | }) |
| 465 | } |
| 466 | } |
| 467 | |
| 468 | func contains(list []int, value int) bool { |
| 469 | for _, elem := range list { |
| 470 | if elem == value { |
| 471 | return true |
| 472 | } |
| 473 | } |
| 474 | return false |
| 475 | } |
| 476 | |
| 477 | func TestPatterns(t *testing.T) { |
| 478 | // Loading all crawlers with go:embed |
| 479 | // some validation happens in UnmarshalJSON. |
| 480 | allCrawlers := Crawlers |
| 481 | |
| 482 | // There are at least 10 crawlers. |
| 483 | if len(allCrawlers) < 10 { |
| 484 | t.Errorf("Number of crawlers must be at least 10, got %d.", len(allCrawlers)) |
| 485 | } |
| 486 | |
| 487 | if IsCrawler(browserUA) { |
| 488 | t.Errorf("Browser UA %q was detected as a crawler.", browserUA) |
| 489 | } |
| 490 | if len(MatchingCrawlers(browserUA)) != 0 { |
| 491 | t.Errorf("MatchingCrawlers found crawlers matching Browser UA %q.", browserUA) |
| 492 | } |
| 493 | |
| 494 | for i, crawler := range allCrawlers { |
| 495 | t.Run(crawler.Pattern, func(t *testing.T) { |
| 496 | fmt.Println(crawler.Pattern) |
| 497 | |
| 498 | for _, instance := range crawler.Instances { |
| 499 | if !IsCrawler(instance) { |
| 500 | t.Errorf("Instance %q is not detected as a crawler.", instance) |
| 501 | } |
| 502 | hits := MatchingCrawlers(instance) |
| 503 | if !contains(hits, i) { |
| 504 | t.Errorf("Crawler with index %d (pattern %q) is not in the list returned by MatchingCrawlers(%q): %v.", i, crawler.Pattern, instance, hits) |
| 505 | } |
| 506 | } |
| 507 | }) |
| 508 | } |
| 509 | } |
| 510 | |
| 511 | func TestFalseNegatives(t *testing.T) { |
| 512 | const browsersURL = "https://raw.githubusercontent.com/microlinkhq/top-user-agents/master/src/index.json" |
| 513 | resp, err := http.Get(browsersURL) |
| 514 | if err != nil { |
| 515 | t.Fatalf("Failed to fetch the list of browser User Agents from %s: %v.", browsersURL, err) |
| 516 | } |
| 517 | |
| 518 | t.Cleanup(func() { |
| 519 | if err := resp.Body.Close(); err != nil { |
| 520 | t.Fatal(err) |
| 521 | } |
| 522 | }) |
| 523 | |
| 524 | var browsers []string |
| 525 | if err := json.NewDecoder(resp.Body).Decode(&browsers); err != nil { |
| 526 | t.Fatalf("Failed to parse the list of browser User Agents: %v.", err) |
| 527 | } |
| 528 | |
| 529 | for _, userAgent := range browsers { |
| 530 | if IsCrawler(userAgent) { |
| 531 | t.Errorf("Browser User Agent %q is recognized as a crawler.", userAgent) |
| 532 | } |
| 533 | indices := MatchingCrawlers(userAgent) |
| 534 | if len(indices) != 0 { |
| 535 | t.Errorf("Browser User Agent %q matches with crawlers %v.", userAgent, indices) |
| 536 | } |
| 537 | } |
| 538 | } |
| 539 | |
| 540 | const ( |
| 541 | crawlerUA = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36 Google (+https://developers.google.com/+/web/snippet/" |
| 542 | browserUA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) obsidian/1.5.3 Chrome/114.0.5735.289 Electron/25.8.1 Safari/537.36" |
| 543 | ) |
| 544 | |
| 545 | func BenchmarkIsCrawlerPositive(b *testing.B) { |
| 546 | b.SetBytes(int64(len(crawlerUA))) |
| 547 | for n := 0; n < b.N; n++ { |
| 548 | if !IsCrawler(crawlerUA) { |
| 549 | b.Fail() |
| 550 | } |
| 551 | } |
| 552 | } |
| 553 | |
| 554 | func BenchmarkMatchingCrawlersPositive(b *testing.B) { |
| 555 | b.SetBytes(int64(len(crawlerUA))) |
| 556 | for n := 0; n < b.N; n++ { |
| 557 | if len(MatchingCrawlers(crawlerUA)) == 0 { |
| 558 | b.Fail() |
| 559 | } |
| 560 | } |
| 561 | } |
| 562 | |
| 563 | func BenchmarkIsCrawlerNegative(b *testing.B) { |
| 564 | b.SetBytes(int64(len(browserUA))) |
| 565 | for n := 0; n < b.N; n++ { |
| 566 | if IsCrawler(browserUA) { |
| 567 | b.Fail() |
| 568 | } |
| 569 | } |
| 570 | } |
| 571 | |
| 572 | func BenchmarkMatchingCrawlersNegative(b *testing.B) { |
| 573 | b.SetBytes(int64(len(browserUA))) |
| 574 | for n := 0; n < b.N; n++ { |
| 575 | if len(MatchingCrawlers(browserUA)) != 0 { |
| 576 | b.Fail() |
| 577 | } |
| 578 | } |
| 579 | } |
| 580 |