re-factored is_manrs_participant to scrape live data from manrs.org w…

…ebsite.

main.py

@@ -3,13 +3,41 @@
 import csv
 import re
 from pathlib import *
+import pandas as pd
+import requests
+from bs4 import BeautifulSoup
 
 manrs_asns = []
 manrs_csv_filename = "manrs.csv"
 canarie_route_filename = "canarie_routes.csv"
 canarie_routes = {}
+manrs_participant_asns = []
 rov = ROV()
 
+
+# Import the table id 'participant-table' from the website https://www.manrs.org/netops/participants/
+# and add put it in a dataframe called manrs
+def is_manrs_participant(origin_asn):
+    origin_asn = str(origin_asn)
+    global manrs_participant_asns
+    if not manrs_participant_asns:
+        url = 'https://www.manrs.org/netops/participants/'
+        r = requests.get(url)
+        soup = BeautifulSoup(r.text, 'html.parser')
+        table = soup.find('table', attrs={'id': 'participant-table'})
+        manrs = pd.read_html(str(table))[0]
+        asns = manrs['ASNs'].tolist()
+        for asn in asns:
+            asn = str(asn)
+            for tmp in asn.split(','):
+                manrs_participant_asns.append(tmp.strip())
+
+    if origin_asn in manrs_participant_asns:
+        return "Yes"
+    else:
+        return "No"
+
+
 def read_manrs_participant_file(filename):
     global manrs_asns
     read_manrs_participant = Path.cwd() / filename
@@ -42,17 +70,17 @@ def get_origin(as_path):
     return origin
 
 
-def is_manrs_participant(asn):
-    global manrs_asns
-    if len(manrs_asns) == 0:
-        read_manrs_participant_file(manrs_csv_filename)
-    asn = str(asn)
-    print(f"asn = {asn} in {manrs_asns}")
-
-    if asn in manrs_asns:
-        return 'Yes'
-    else:
-        return 'No'
+# def is_manrs_participant(asn):
+#     global manrs_asns
+#     if len(manrs_asns) == 0:
+#         read_manrs_participant_file(manrs_csv_filename)
+#     asn = str(asn)
+#     print(f"asn = {asn} in {manrs_asns}")
+#
+#     if asn in manrs_asns:
+#         return 'Yes'
+#     else:
+#         return 'No'
 
 def rov_status(prefix, asn):
     state = rov.check(prefix, asn)
@@ -83,12 +111,6 @@ def check_routes(report_filename):
                              'manrs participant': manrs_part})
 
 
-    for prefix in canarie_routes:
-        asn = canarie_routes[prefix]
-        state = rov_status(prefix, asn)
-
-
-
 def init_rov():
     # rov.download_databases()
     rov.load_databases()
@@ -102,9 +124,9 @@ def print_hi(name):
 
 # Press the green button in the gutter to run the script.
 if __name__ == '__main__':
-    init_rov()
-    read_canarie_routes(canarie_route_filename)
-    check_routes('ca_report.csv')
+   init_rov()
+   read_canarie_routes(canarie_route_filename)
+   check_routes('ca_report2.csv')
 
 
 # See PyCharm help at https://www.jetbrains.com/help/pycharm/