diff --git a/app/api_json.py b/app/api_json.py index ff1e239..c6cc042 100644 --- a/app/api_json.py +++ b/app/api_json.py @@ -38,11 +38,15 @@ # academic research and education purposes is subject to the conditions and # copyright notices in the source code files and in the included LICENSE file. +import json + from ipaddress import ip_network import elasticsearch from flask import Blueprint, request, current_app -import requests, json +import requests + +from app.config.nested_tag_descriptions import NESTED_TAG_DICT from app.elastic import getElastic from app.utils import handle_exception, post_process, validate_event_id from app.GripException import ValidationError @@ -53,6 +57,8 @@ def json_tags(): r = requests.get(current_app.config['META_SERVICE'] + "/tags") data = json.loads(r.content.decode('utf-8')) + + data['nested_tag_descriptions'] = NESTED_TAG_DICT return post_process(data), 200 @bp.route('/asndrop', methods=['GET']) @@ -78,10 +84,12 @@ def json_blocklist(): @bp.route('/event/id/', methods=['GET']) def json_event_by_id(evid): + nested = request.args.get('nested', 'false').lower() == 'true' try: es = getElastic() validate_event_id(evid) - pending = es.getEventById(evid) + + pending = es.get_event_by_id(evid, nested) return post_process(pending), 200 except elasticsearch.exceptions.NotFoundError: @@ -104,10 +112,12 @@ def json_search_events(): @bp.route('/pfx_event/id//', methods=['GET']) def json_pfx_event_by_id(evid, prefix): + nested = request.args.get('nested', 'false').lower() == 'true' + try: es = getElastic() validate_event_id(evid) - fullev = es.getEventById(evid) + fullev = es.get_event_by_id(evid, nested) replaced = prefix.replace("-", "/") search = replaced.split("_") diff --git a/app/config/nested_tag_descriptions.py b/app/config/nested_tag_descriptions.py new file mode 100644 index 0000000..9d65e72 --- /dev/null +++ b/app/config/nested_tag_descriptions.py @@ -0,0 +1,14 @@ +NESTED_TAG_DICT = { + "all-newcomer-exact-record": "all newcomer origins have exact IRR records for the announced prefix (subprefix)", + "all-newcomer-more-specific-record": "all newcomer origins have less specific IRR records for the announced prefix (subprefix)", + "all-newcomer-no-record": "all newcomer origins have no IRR records for the announced prefix (subprefix)", + "some-newcomer-exact-record": "some newcomer origins have exact IRR records for the announced prefix (subprefix)", + "some-newcomer-more-specific-record": "some newcomer origins have less specific IRR records for the announced prefix (subprefix)", + "some-newcomer-no-record": "some newcomer origins have no IRR records for the announced prefix (subprefix)", + "all-oldcomer-exact-record": "all oldcomer origins have exact IRR records for the announced prefix (subprefix)", + "all-oldcomer-more-specific-record": "all oldcomer origins have less specific IRR records for the announced prefix (subprefix)", + "all-oldcomer-no-record": "all oldcomer origins have no IRR records for the announced prefix (subprefix)", + "some-oldcomer-exact-record": "some oldcomer origins have exact IRR records for the announced prefix (subprefix)", + "some-oldcomer-more-specific-record": "some oldcomer origins have less specific IRR records for the announced prefix (subprefix)", + "some-oldcomer-no-record": "some oldcomer origins have no IRR records for the announced prefix (subprefix)", +} diff --git a/app/elastic.py b/app/elastic.py index b6cf1b7..b18dfad 100644 --- a/app/elastic.py +++ b/app/elastic.py @@ -42,8 +42,9 @@ from datetime import datetime import re, time +from collections import defaultdict -from flask import current_app, g, request, jsonify +from flask import current_app, g from app.GripException import ValidationError @@ -307,26 +308,42 @@ def __init__(self, nodes, api_key_id, api_key_secret): if not self.es.ping(): raise ValueError("Failed to connect to ElasticSearch") - def getEventById(self, evid): - evparams = evid.split('-') - if len(evparams) != 3: + def get_event_by_id(self, event_id, nested = False): + event_params = event_id.split('-') + if len(event_params) != 3: err_str = "Invalid event ID format -- should be --" raise ValidationError(err_str) - - evtype = evparams[0] - + + event_type = event_params[0] + try: - evts = datetime.fromtimestamp(int(evparams[1])) - datestr = datetime.strftime(evts, "%Y-%m") + evts = datetime.fromtimestamp(int(event_params[1])) + date_str = datetime.strftime(evts, "%Y-%m") except: err_str = "Invalid timestamp in event ID -- should be a unix timestamp" raise ValidationError(err_str) - indexname = "observatory-v4-query-events-{}-{}".format( - evtype, datestr) - result = self.es.get(index=indexname, id=evid) - event = enhance_pfxevents_for_event(result['_source']) - return event + indexname = f"observatory-v4-query-events-{event_type}-{date_str}" + result = self.es.get(index=indexname, id=event_id) + event_response = enhance_pfxevents_for_event(result['_source']) + + if nested: + tag_list = result['_source']['summary']['tags'] + tag_families = defaultdict(list) + + # The groupings for tag_families are based on the + # irr-[IRR]-[common_suffix] families of tags + + for tag in tag_list: + if tag['name'].startswith("irr-"): + tag_suffix = tag.split("-", 2)[-1] + tag_families[tag_suffix].append(tag) + else: + tag_families["other"].append(tag) + + event_response['summary']['tags'] = tag_families + + return event_response def lookupEvents(self, queryparams): @@ -342,9 +359,9 @@ def lookupEvents(self, queryparams): full = queryparams.get("full") if debug is not None: - index = "observatory-v4-test-events-{}-*".format(event_type) + index = f"observatory-v4-test-events-{event_type}-*" else: - index = "observatory-v4-query-events-{}-*".format(event_type) + index = f"observatory-v4-query-events-{event_type}-*" kwargs = {'from': start, 'size': size, 'sort': "view_ts:desc"}