.github
1 year ago
.gitignore
1 year ago
.travis.yml
2 years ago
LICENSE
3 years ago
MANIFEST.in
2 years ago
README.md
1 year ago
__init__.py
1 year ago
composer.json
3 years ago
crawler-user-agents.json
1 year ago
format.js
1 year ago
go.mod
2 years ago
go.sum
2 years ago
index.d.ts
3 years ago
main.php
3 years ago
package-lock.json
2 years ago
package.json
2 years ago
pyproject.toml
1 year ago
test_harness.py
1 year ago
test_validation.py
3 years ago
validate.go
1 year ago
validate.php
3 years ago
validate.py
2 years ago
validate_test.go
1 year ago
validate.py
110 lines
| 1 | """ |
| 2 | Validate JSON to ensure that patterns all work |
| 3 | """ |
| 4 | from __future__ import print_function |
| 5 | |
| 6 | import json |
| 7 | import re |
| 8 | from collections import Counter |
| 9 | import datetime |
| 10 | |
| 11 | from jsonschema import validate |
| 12 | |
| 13 | |
| 14 | JSON_SCHEMA = { |
| 15 | "type": "array", |
| 16 | "items": { |
| 17 | "type": "object", |
| 18 | "properties": { |
| 19 | "pattern": {"type": "string"}, # required |
| 20 | "instances": {"type": "array"}, # required |
| 21 | "url": {"type": "string"}, # optional |
| 22 | "description": {"type": "string"}, # optional |
| 23 | "addition_date": {"type": "string"}, # optional |
| 24 | "depends_on": {"type": "array"} # allows an instance to match twice |
| 25 | }, |
| 26 | "required": ["pattern", "instances"] |
| 27 | } |
| 28 | } |
| 29 | |
| 30 | |
| 31 | def main(): |
| 32 | with open('crawler-user-agents.json') as f: |
| 33 | json_data = json.load(f) |
| 34 | |
| 35 | # check format using JSON Schema |
| 36 | validate(json_data, JSON_SCHEMA) |
| 37 | |
| 38 | # check for simple duplicates |
| 39 | pattern_counts = Counter(entry['pattern'] for entry in json_data) |
| 40 | for pattern, count in pattern_counts.most_common(): |
| 41 | if count > 1: |
| 42 | raise ValueError('Pattern {!r} appears {} times'.format(pattern, |
| 43 | count)) |
| 44 | |
| 45 | # check for duplicates with different capitalization |
| 46 | pattern_counts = Counter(entry['pattern'].lower() for entry in json_data) |
| 47 | for pattern, count in pattern_counts.most_common(): |
| 48 | if count > 1: |
| 49 | raise ValueError('Pattern {!r} is duplicated {} times with different capitalization' |
| 50 | .format(pattern, count)) |
| 51 | |
| 52 | # checks that no pattern contains unescaped slash / |
| 53 | for entry in json_data: |
| 54 | pattern = entry['pattern'] |
| 55 | if re.search('[^\\\\]/', pattern): |
| 56 | raise ValueError('Pattern {!r} has an unescaped slash character'.format(pattern)) |
| 57 | |
| 58 | # check that no pattern contains unescaped dot . |
| 59 | for entry in json_data: |
| 60 | pattern = entry['pattern'] |
| 61 | if re.search('[^\\\\]\\.', pattern): |
| 62 | raise ValueError('Pattern {!r} has an unescaped dot character'.format(pattern)) |
| 63 | |
| 64 | # check that we match the given instances |
| 65 | num_instances = 0 |
| 66 | for entry in json_data: |
| 67 | pattern = entry['pattern'] |
| 68 | |
| 69 | # assert that field "addition_date" has format "2019/12/23", |
| 70 | if 'addition_date' in entry: |
| 71 | if not re.match(r'\d{4}/\d{2}/\d{2}', entry['addition_date']): |
| 72 | raise ValueError('addition_date {!r} has invalid format'.format(entry['addition_date'])) |
| 73 | # parse the date with datetime |
| 74 | datetime.datetime.strptime(entry['addition_date'], '%Y/%m/%d') |
| 75 | |
| 76 | # canonicalize entry |
| 77 | if 'depends_on' not in entry: entry['depends_on'] = [] |
| 78 | |
| 79 | # check that we have only the rights properties (not handled by default in module jsonschema) |
| 80 | assert set([str(x) for x in entry.keys()]).issubset(set(JSON_SCHEMA['items']['properties'].keys())), "the entry contains unknown properties" |
| 81 | instances = entry.get('instances') |
| 82 | if instances: |
| 83 | # check that there is no duplicate |
| 84 | if not len(instances) == len(set(instances)): |
| 85 | raise Exception("duplicate instances in "+pattern) |
| 86 | for instance in instances: |
| 87 | num_instances += 1 |
| 88 | if not re.search(pattern, instance): |
| 89 | raise ValueError('Pattern {!r} misses instance {!r}' |
| 90 | .format(pattern, instance)) |
| 91 | |
| 92 | |
| 93 | # Make sure we have at least one pattern |
| 94 | if len(json_data) < 1: |
| 95 | raise Exception("no pattern") |
| 96 | |
| 97 | # Check for patterns that match other patterns |
| 98 | for entry1 in json_data: |
| 99 | for entry2 in json_data: |
| 100 | if entry1 != entry2 and re.search(entry1['pattern'], |
| 101 | entry2['pattern'],re.IGNORECASE): |
| 102 | raise ValueError('Pattern {!r} is a subset of {!r}' |
| 103 | .format(entry2['pattern'], entry1['pattern'])) |
| 104 | |
| 105 | print('Validation passed') |
| 106 | |
| 107 | |
| 108 | if __name__ == '__main__': |
| 109 | main() |
| 110 |