Add functionality for nested tags

app/api_json.py

@@ -38,11 +38,15 @@
 # academic research and education purposes is subject to the conditions and
 # copyright notices in the source code files and in the included LICENSE file.
 
+import json
+
 from ipaddress import ip_network
 import elasticsearch
 from flask import Blueprint, request, current_app
-import requests, json
+import requests
+
 
+from app.config.nested_tag_descriptions import NESTED_TAG_DICT
 from app.elastic import getElastic
 from app.utils import handle_exception, post_process, validate_event_id
 from app.GripException import ValidationError
@@ -53,6 +57,8 @@
 def json_tags():
     r = requests.get(current_app.config['META_SERVICE'] + "/tags")
     data = json.loads(r.content.decode('utf-8'))
+
+    data['nested_tag_descriptions'] = NESTED_TAG_DICT
     return post_process(data), 200
 
 @bp.route('/asndrop', methods=['GET'])
@@ -78,10 +84,12 @@ def json_blocklist():
 
 @bp.route('/event/id/<evid>', methods=['GET'])
 def json_event_by_id(evid):
+    nested = request.args.get('nested', 'false').lower() == 'true'
     try:
         es = getElastic()
         validate_event_id(evid)
-        pending = es.getEventById(evid)
+
+        pending = es.get_event_by_id(evid, nested)
         return post_process(pending), 200
 
     except elasticsearch.exceptions.NotFoundError:
@@ -104,10 +112,12 @@ def json_search_events():
 
 @bp.route('/pfx_event/id/<evid>/<prefix>', methods=['GET'])
 def json_pfx_event_by_id(evid, prefix):
+    nested = request.args.get('nested', 'false').lower() == 'true'
+
     try:
         es = getElastic()
         validate_event_id(evid)
-        fullev = es.getEventById(evid)
+        fullev = es.get_event_by_id(evid, nested)
 
         replaced = prefix.replace("-", "/")
         search = replaced.split("_")

app/config/nested_tag_descriptions.py

@@ -0,0 +1,14 @@
+NESTED_TAG_DICT = {
+    "all-newcomer-exact-record": "all newcomer origins have exact IRR records for the announced prefix (subprefix)",
+    "all-newcomer-more-specific-record": "all newcomer origins have less specific IRR records for the announced prefix (subprefix)",
+    "all-newcomer-no-record": "all newcomer origins have no IRR records for the announced prefix (subprefix)",
+    "some-newcomer-exact-record": "some newcomer origins have exact IRR records for the announced prefix (subprefix)",
+    "some-newcomer-more-specific-record": "some newcomer origins have less specific IRR records for the announced prefix (subprefix)",
+    "some-newcomer-no-record": "some newcomer origins have no IRR records for the announced prefix (subprefix)",
+    "all-oldcomer-exact-record": "all oldcomer origins have exact IRR records for the announced prefix (subprefix)",
+    "all-oldcomer-more-specific-record": "all oldcomer origins have less specific IRR records for the announced prefix (subprefix)",
+    "all-oldcomer-no-record": "all oldcomer origins have no IRR records for the announced prefix (subprefix)",
+    "some-oldcomer-exact-record": "some oldcomer origins have exact IRR records for the announced prefix (subprefix)",
+    "some-oldcomer-more-specific-record": "some oldcomer origins have less specific IRR records for the announced prefix (subprefix)",
+    "some-oldcomer-no-record": "some oldcomer origins have no IRR records for the announced prefix (subprefix)",
+}

app/elastic.py

@@ -42,8 +42,9 @@
 from datetime import datetime
 
 import re, time
+from collections import defaultdict
 
-from flask import current_app, g, request, jsonify
+from flask import current_app, g
 
 from app.GripException import ValidationError
 
@@ -307,26 +308,42 @@ def __init__(self, nodes, api_key_id, api_key_secret):
         if not self.es.ping():
             raise ValueError("Failed to connect to ElasticSearch")
 
-    def getEventById(self, evid):
-        evparams = evid.split('-')
-        if len(evparams) != 3:
+    def get_event_by_id(self, event_id, nested = False):
+        event_params = event_id.split('-')
+        if len(event_params) != 3:
             err_str = "Invalid event ID format -- should be <evtype>-<timestamp>-<aslist>"
             raise ValidationError(err_str)
-         
-        evtype = evparams[0]
-        
+
+        event_type = event_params[0]
+
         try:
-            evts = datetime.fromtimestamp(int(evparams[1]))
-            datestr = datetime.strftime(evts, "%Y-%m")
+            evts = datetime.fromtimestamp(int(event_params[1]))
+            date_str = datetime.strftime(evts, "%Y-%m")
         except:
             err_str = "Invalid timestamp in event ID -- should be a unix timestamp"
             raise ValidationError(err_str)
 
-        indexname = "observatory-v4-query-events-{}-{}".format(
-                evtype, datestr)
-        result = self.es.get(index=indexname, id=evid)
-        event = enhance_pfxevents_for_event(result['_source'])
-        return event
+        indexname = f"observatory-v4-query-events-{event_type}-{date_str}"
+        result = self.es.get(index=indexname, id=event_id)
+        event_response = enhance_pfxevents_for_event(result['_source'])
+
+        if nested:
+            tag_list = result['_source']['summary']['tags']
+            tag_families = defaultdict(list)
+
+            # The groupings for tag_families are based on the 
+            # irr-[IRR]-[common_suffix] families of tags
+
+            for tag in tag_list:
+                if tag['name'].startswith("irr-"):
+                    tag_suffix = tag.split("-", 2)[-1]
+                    tag_families[tag_suffix].append(tag)
+                else:
+                    tag_families["other"].append(tag)
+
+            event_response['summary']['tags'] = tag_families
+
+        return event_response
 
     def lookupEvents(self, queryparams):
 
@@ -342,9 +359,9 @@ def lookupEvents(self, queryparams):
         full = queryparams.get("full")
 
         if debug is not None:
-            index = "observatory-v4-test-events-{}-*".format(event_type)
+            index = f"observatory-v4-test-events-{event_type}-*"
         else:
-            index = "observatory-v4-query-events-{}-*".format(event_type)
+            index = f"observatory-v4-query-events-{event_type}-*"
 
         kwargs = {'from': start, 'size': size,
                 'sort': "view_ts:desc"}