From 7f83be1155d0c191615e4a7a5a2350da691b5b9e Mon Sep 17 00:00:00 2001 From: VijayrajS Date: Sat, 12 Sep 2026 17:58:26 -0700 Subject: [PATCH 1/5] Added function for nested tags --- app/api_json.py | 13 +++++++++---- app/elastic.py | 49 +++++++++++++++++++++++++++++++++---------------- 2 files changed, 42 insertions(+), 20 deletions(-) diff --git a/app/api_json.py b/app/api_json.py index ff1e239..526f028 100644 --- a/app/api_json.py +++ b/app/api_json.py @@ -77,11 +77,15 @@ def json_blocklist(): return post_process(data), 200 @bp.route('/event/id/', methods=['GET']) -def json_event_by_id(evid): +def json_event_by_id(evid, version='v1'): + # v2 API returns tags in a nested format, v1 has to eventually be removed + # after transition + try: es = getElastic() validate_event_id(evid) - pending = es.getEventById(evid) + + pending = es.get_event_by_id(evid, version) return post_process(pending), 200 except elasticsearch.exceptions.NotFoundError: @@ -103,11 +107,12 @@ def json_search_events(): return post_process(pending), 200 @bp.route('/pfx_event/id//', methods=['GET']) -def json_pfx_event_by_id(evid, prefix): +def json_pfx_event_by_id(evid, prefix, version='v1'): + # v2 API returns tags in a nested format, v1 has to eventually be removed try: es = getElastic() validate_event_id(evid) - fullev = es.getEventById(evid) + fullev = es.get_event_by_id(evid, version) replaced = prefix.replace("-", "/") search = replaced.split("_") diff --git a/app/elastic.py b/app/elastic.py index b6cf1b7..210eb57 100644 --- a/app/elastic.py +++ b/app/elastic.py @@ -42,8 +42,9 @@ from datetime import datetime import re, time +from collections import defaultdict -from flask import current_app, g, request, jsonify +from flask import current_app, g from app.GripException import ValidationError @@ -307,26 +308,42 @@ def __init__(self, nodes, api_key_id, api_key_secret): if not self.es.ping(): raise ValueError("Failed to connect to ElasticSearch") - def getEventById(self, evid): - evparams = evid.split('-') - if len(evparams) != 3: + def get_event_by_id(self, event_id, version = 'v1'): + event_params = event_id.split('-') + if len(event_params) != 3: err_str = "Invalid event ID format -- should be --" raise ValidationError(err_str) - - evtype = evparams[0] - + + event_type = event_params[0] + try: - evts = datetime.fromtimestamp(int(evparams[1])) - datestr = datetime.strftime(evts, "%Y-%m") + evts = datetime.fromtimestamp(int(event_params[1])) + date_str = datetime.strftime(evts, "%Y-%m") except: err_str = "Invalid timestamp in event ID -- should be a unix timestamp" raise ValidationError(err_str) - indexname = "observatory-v4-query-events-{}-{}".format( - evtype, datestr) - result = self.es.get(index=indexname, id=evid) - event = enhance_pfxevents_for_event(result['_source']) - return event + indexname = f"observatory-v4-query-events-{event_type}-{date_str}" + result = self.es.get(index=indexname, id=event_id) + event_response = enhance_pfxevents_for_event(result['_source']) + + if version == 'v2': + tag_list = result['_source']['summary']['tags'] + tag_families = defaultdict(list) + + # The groupings for tag_families are based on the + # irr-[IRR]-[common_suffix] families of tags + + for tag in tag_list: + if tag.startswith("irr-"): + tag_suffix = tag.split("-", 2)[-1] + tag_families[tag_suffix].append(tag) + else: + tag_families["other"].append(tag) + + event_response['summary']['tags'] = tag_families + + return event_response def lookupEvents(self, queryparams): @@ -342,9 +359,9 @@ def lookupEvents(self, queryparams): full = queryparams.get("full") if debug is not None: - index = "observatory-v4-test-events-{}-*".format(event_type) + index = f"observatory-v4-test-events-{event_type}-*" else: - index = "observatory-v4-query-events-{}-*".format(event_type) + index = f"observatory-v4-query-events-{event_type}-*" kwargs = {'from': start, 'size': size, 'sort': "view_ts:desc"} From a37b020c1f2170cfb5990ee231c1f57191ffb7f5 Mon Sep 17 00:00:00 2001 From: VijayrajS Date: Sat, 12 Sep 2026 18:13:47 -0700 Subject: [PATCH 2/5] Added nested tag descriptions --- app/config/nested_tag_descriptions.json | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 app/config/nested_tag_descriptions.json diff --git a/app/config/nested_tag_descriptions.json b/app/config/nested_tag_descriptions.json new file mode 100644 index 0000000..0d3bf93 --- /dev/null +++ b/app/config/nested_tag_descriptions.json @@ -0,0 +1,16 @@ +{ + "nested_tag_descriptions": { + "all-newcomer-exact-record": "all newcomer origins have exact IRR records for the announced prefix (subprefix)", + "all-newcomer-more-specific-record": "all newcomer origins have less specific IRR records for the announced prefix (subprefix)", + "all-newcomer-no-record": "all newcomer origins have no IRR records for the announced prefix (subprefix)", + "some-newcomer-exact-record": "some newcomer origins have exact IRR records for the announced prefix (subprefix)", + "some-newcomer-more-specific-record": "some newcomer origins have less specific IRR records for the announced prefix (subprefix)", + "some-newcomer-no-record": "some newcomer origins have no IRR records for the announced prefix (subprefix)", + "all-oldcomer-exact-record": "all oldcomer origins have exact IRR records for the announced prefix (subprefix)", + "all-oldcomer-more-specific-record": "all oldcomer origins have less specific IRR records for the announced prefix (subprefix)", + "all-oldcomer-no-record": "all oldcomer origins have no IRR records for the announced prefix (subprefix)", + "some-oldcomer-exact-record": "some oldcomer origins have exact IRR records for the announced prefix (subprefix)", + "some-oldcomer-more-specific-record": "some oldcomer origins have less specific IRR records for the announced prefix (subprefix)", + "some-oldcomer-no-record": "some oldcomer origins have no IRR records for the announced prefix (subprefix)" + } +} \ No newline at end of file From e39925ce230d33ad32f4f3777d479031e1d098ab Mon Sep 17 00:00:00 2001 From: VijayrajS Date: Sat, 12 Sep 2026 18:19:51 -0700 Subject: [PATCH 3/5] Added nested tag descriptions to /tags endpoint --- app/api_json.py | 4 ++++ app/config/nested_tag_descriptions.json | 16 ---------------- app/config/nested_tag_descriptions.py | 14 ++++++++++++++ 3 files changed, 18 insertions(+), 16 deletions(-) delete mode 100644 app/config/nested_tag_descriptions.json create mode 100644 app/config/nested_tag_descriptions.py diff --git a/app/api_json.py b/app/api_json.py index 526f028..88383ef 100644 --- a/app/api_json.py +++ b/app/api_json.py @@ -43,6 +43,8 @@ from flask import Blueprint, request, current_app import requests, json +from app.config.nested_tag_descriptions import NESTED_TAG_DICT + from app.elastic import getElastic from app.utils import handle_exception, post_process, validate_event_id from app.GripException import ValidationError @@ -53,6 +55,8 @@ def json_tags(): r = requests.get(current_app.config['META_SERVICE'] + "/tags") data = json.loads(r.content.decode('utf-8')) + + data['nested_tag_descriptions'] = NESTED_TAG_DICT return post_process(data), 200 @bp.route('/asndrop', methods=['GET']) diff --git a/app/config/nested_tag_descriptions.json b/app/config/nested_tag_descriptions.json deleted file mode 100644 index 0d3bf93..0000000 --- a/app/config/nested_tag_descriptions.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "nested_tag_descriptions": { - "all-newcomer-exact-record": "all newcomer origins have exact IRR records for the announced prefix (subprefix)", - "all-newcomer-more-specific-record": "all newcomer origins have less specific IRR records for the announced prefix (subprefix)", - "all-newcomer-no-record": "all newcomer origins have no IRR records for the announced prefix (subprefix)", - "some-newcomer-exact-record": "some newcomer origins have exact IRR records for the announced prefix (subprefix)", - "some-newcomer-more-specific-record": "some newcomer origins have less specific IRR records for the announced prefix (subprefix)", - "some-newcomer-no-record": "some newcomer origins have no IRR records for the announced prefix (subprefix)", - "all-oldcomer-exact-record": "all oldcomer origins have exact IRR records for the announced prefix (subprefix)", - "all-oldcomer-more-specific-record": "all oldcomer origins have less specific IRR records for the announced prefix (subprefix)", - "all-oldcomer-no-record": "all oldcomer origins have no IRR records for the announced prefix (subprefix)", - "some-oldcomer-exact-record": "some oldcomer origins have exact IRR records for the announced prefix (subprefix)", - "some-oldcomer-more-specific-record": "some oldcomer origins have less specific IRR records for the announced prefix (subprefix)", - "some-oldcomer-no-record": "some oldcomer origins have no IRR records for the announced prefix (subprefix)" - } -} \ No newline at end of file diff --git a/app/config/nested_tag_descriptions.py b/app/config/nested_tag_descriptions.py new file mode 100644 index 0000000..9d65e72 --- /dev/null +++ b/app/config/nested_tag_descriptions.py @@ -0,0 +1,14 @@ +NESTED_TAG_DICT = { + "all-newcomer-exact-record": "all newcomer origins have exact IRR records for the announced prefix (subprefix)", + "all-newcomer-more-specific-record": "all newcomer origins have less specific IRR records for the announced prefix (subprefix)", + "all-newcomer-no-record": "all newcomer origins have no IRR records for the announced prefix (subprefix)", + "some-newcomer-exact-record": "some newcomer origins have exact IRR records for the announced prefix (subprefix)", + "some-newcomer-more-specific-record": "some newcomer origins have less specific IRR records for the announced prefix (subprefix)", + "some-newcomer-no-record": "some newcomer origins have no IRR records for the announced prefix (subprefix)", + "all-oldcomer-exact-record": "all oldcomer origins have exact IRR records for the announced prefix (subprefix)", + "all-oldcomer-more-specific-record": "all oldcomer origins have less specific IRR records for the announced prefix (subprefix)", + "all-oldcomer-no-record": "all oldcomer origins have no IRR records for the announced prefix (subprefix)", + "some-oldcomer-exact-record": "some oldcomer origins have exact IRR records for the announced prefix (subprefix)", + "some-oldcomer-more-specific-record": "some oldcomer origins have less specific IRR records for the announced prefix (subprefix)", + "some-oldcomer-no-record": "some oldcomer origins have no IRR records for the announced prefix (subprefix)", +} From c90bc60fac486aee602a629457ee9b7972645c3c Mon Sep 17 00:00:00 2001 From: VijayrajS Date: Sat, 12 Sep 2026 20:11:04 -0700 Subject: [PATCH 4/5] Tag has an attribute name --- app/elastic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/elastic.py b/app/elastic.py index 210eb57..b714673 100644 --- a/app/elastic.py +++ b/app/elastic.py @@ -335,7 +335,7 @@ def get_event_by_id(self, event_id, version = 'v1'): # irr-[IRR]-[common_suffix] families of tags for tag in tag_list: - if tag.startswith("irr-"): + if tag['name'].startswith("irr-"): tag_suffix = tag.split("-", 2)[-1] tag_families[tag_suffix].append(tag) else: From 9dcf80c765c889b6874654e694f7245265ccf80f Mon Sep 17 00:00:00 2001 From: VijayrajS Date: Mon, 14 Sep 2026 20:56:53 -0700 Subject: [PATCH 5/5] Change version to nested --- app/api_json.py | 23 ++++++++++++----------- app/elastic.py | 4 ++-- 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/app/api_json.py b/app/api_json.py index 88383ef..c6cc042 100644 --- a/app/api_json.py +++ b/app/api_json.py @@ -38,13 +38,15 @@ # academic research and education purposes is subject to the conditions and # copyright notices in the source code files and in the included LICENSE file. +import json + from ipaddress import ip_network import elasticsearch from flask import Blueprint, request, current_app -import requests, json +import requests -from app.config.nested_tag_descriptions import NESTED_TAG_DICT +from app.config.nested_tag_descriptions import NESTED_TAG_DICT from app.elastic import getElastic from app.utils import handle_exception, post_process, validate_event_id from app.GripException import ValidationError @@ -81,15 +83,13 @@ def json_blocklist(): return post_process(data), 200 @bp.route('/event/id/', methods=['GET']) -def json_event_by_id(evid, version='v1'): - # v2 API returns tags in a nested format, v1 has to eventually be removed - # after transition - +def json_event_by_id(evid): + nested = request.args.get('nested', 'false').lower() == 'true' try: es = getElastic() validate_event_id(evid) - - pending = es.get_event_by_id(evid, version) + + pending = es.get_event_by_id(evid, nested) return post_process(pending), 200 except elasticsearch.exceptions.NotFoundError: @@ -111,12 +111,13 @@ def json_search_events(): return post_process(pending), 200 @bp.route('/pfx_event/id//', methods=['GET']) -def json_pfx_event_by_id(evid, prefix, version='v1'): - # v2 API returns tags in a nested format, v1 has to eventually be removed +def json_pfx_event_by_id(evid, prefix): + nested = request.args.get('nested', 'false').lower() == 'true' + try: es = getElastic() validate_event_id(evid) - fullev = es.get_event_by_id(evid, version) + fullev = es.get_event_by_id(evid, nested) replaced = prefix.replace("-", "/") search = replaced.split("_") diff --git a/app/elastic.py b/app/elastic.py index b714673..b18dfad 100644 --- a/app/elastic.py +++ b/app/elastic.py @@ -308,7 +308,7 @@ def __init__(self, nodes, api_key_id, api_key_secret): if not self.es.ping(): raise ValueError("Failed to connect to ElasticSearch") - def get_event_by_id(self, event_id, version = 'v1'): + def get_event_by_id(self, event_id, nested = False): event_params = event_id.split('-') if len(event_params) != 3: err_str = "Invalid event ID format -- should be --" @@ -327,7 +327,7 @@ def get_event_by_id(self, event_id, version = 'v1'): result = self.es.get(index=indexname, id=event_id) event_response = enhance_pfxevents_for_event(result['_source']) - if version == 'v2': + if nested: tag_list = result['_source']['summary']['tags'] tag_families = defaultdict(list)