From cf6c1c13b004c8d3dd1c471caf7cbaedb3d971d9 Mon Sep 17 00:00:00 2001 From: Alex Stuart Date: Mon, 20 May 2024 15:39:46 +0100 Subject: [PATCH] Remove charting directory See ukf/ukf-meta#410 for details --- charting/Months.pm | 33 ------- charting/by_registrar.py | 91 -------------------- charting/fetch.pl | 36 -------- charting/just_ours.xsl | 44 ---------- charting/mdui.pl | 36 -------- charting/requestedattribute.pl | 43 ---------- charting/requestedattribute.xsl | 37 -------- charting/saml2.pl | 68 --------------- charting/saml2.xsl | 52 ----------- charting/scopes.pl | 50 ----------- charting/scopes.xsl | 46 ---------- charting/sizes.pl | 104 ---------------------- charting/statistics_mdui.xsl | 148 -------------------------------- 13 files changed, 788 deletions(-) delete mode 100755 charting/Months.pm delete mode 100755 charting/by_registrar.py delete mode 100755 charting/fetch.pl delete mode 100644 charting/just_ours.xsl delete mode 100755 charting/mdui.pl delete mode 100755 charting/requestedattribute.pl delete mode 100644 charting/requestedattribute.xsl delete mode 100755 charting/saml2.pl delete mode 100644 charting/saml2.xsl delete mode 100755 charting/scopes.pl delete mode 100644 charting/scopes.xsl delete mode 100755 charting/sizes.pl delete mode 100644 charting/statistics_mdui.xsl diff --git a/charting/Months.pm b/charting/Months.pm deleted file mode 100755 index 86e605d7..00000000 --- a/charting/Months.pm +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/perl -w - -# -# Months.pm -# -use File::stat; -use Date::Manip::Date; -use Date::Manip::Delta; - -@months = ( ); - -$date = new Date::Manip::Date; -$date->parse_date('2006-12-01') && die('could not parse base date'); - -$now = new Date::Manip::Date; -$now->parse('now') && die('could not parse now'); - -$oneMonth = new Date::Manip::Delta; -$oneMonth->parse("1 month") && die('could not parse delta'); - -# -# Enumerate all month beginnings that have already happened, then stop. -# -while ($date->cmp($now) < 0) { - local $dateStr = $date->printf('%Y-%m'); - # print "date string $dateStr\n"; - push @months, $dateStr; - - # move on one month - $date = $date->calc($oneMonth); -} - -1; diff --git a/charting/by_registrar.py b/charting/by_registrar.py deleted file mode 100755 index d433877a..00000000 --- a/charting/by_registrar.py +++ /dev/null @@ -1,91 +0,0 @@ -#!/usr/bin/env python - -''' -Analyse a SAML metadata file and build a histogram of entities binned against -their registrar. -''' - -from xml.dom.minidom import parse -from urllib import urlopen -from datetime import date -import sys - -REGISTRAR_NAME = { - - # eduGAIN participants - "http://eduid.at": "AT", - "http://federation.belnet.be/": "BE", - "http://cafe.rnp.br": "BR", - "http://www.canarie.ca": "CA", - "http://cofre.reuna.cl/": "CL", - "https://www.carsi.edu.cn": "CN", - "http://www.srce.hr": "HR", - "http://www.eduid.cz/": "CZ", - "https://www.wayf.dk": "DK", - "http://www.csc.fi/haka": "FI", - "https://federation.renater.fr/": "FR", - "https://www.aai.dfn.de": "DE", - "http://aai.grnet.gr/": "GR", - "http://eduid.hu": "HU", - "http://www.heanet.ie": "IE", - "http://www.idem.garr.it/": "IT", - "http://laife.lanet.lv/": "LV", - "http://feide.no/": "NO", - "http://aai.arnes.si": "SI", - "http://www.rediris.es/": "ES", - "http://www.swamid.se/": "SE", - "http://rr.aai.switch.ch/": "CH", - "http://www.surfconext.nl/": "NL", - "http://ukfederation.org.uk": "UK", - - # Joining eduGAIN - "http://aai.pionier.net.pl": "PL", - - # not yet eduGAIN members - "https://incommon.org": "US", -} - -def regAuth(uri): - ''' - Returns a short registrar code, or the long authority URI if none is available. - ''' - try: - return REGISTRAR_NAME[uri] - except KeyError: - return uri - -def display(infile, split): - doc = parse(infile) - - # Pull out all of the RegistrationInfo elements, one per entity - registrationInfos = doc.getElementsByTagNameNS("urn:oasis:names:tc:SAML:metadata:rpi", - "RegistrationInfo") - - counts = dict(); - - for info in registrationInfos: - auth = regAuth(info.getAttribute("registrationAuthority")) - try: - counts[auth] += 1 - except KeyError: - counts[auth] = 1 - - counts = sorted(counts.items(), key=lambda item: item[1], reverse=True) - - first_counts = counts[0:split] - rest_counts = counts[split:] - for e in first_counts: - print "%10s: %d" % (e[0], e[1]) - print "%10s: %d" % ("other", sum([e[1] for e in rest_counts])) - -if len(sys.argv) == 2: - display(sys.argv[1], 9) -else: - cache_file = date.today().strftime("cache/%Y-%m.xml") - print "Most recent monthly UK federation production aggregate (%s):" % (cache_file) - display(cache_file, 9) - - print - - print "Current eduGAIN production aggregate:" - display(urlopen("https://mds.edugain.org/edugain-v2.xml"), 9) diff --git a/charting/fetch.pl b/charting/fetch.pl deleted file mode 100755 index 47d3e51e..00000000 --- a/charting/fetch.pl +++ /dev/null @@ -1,36 +0,0 @@ -#!/usr/bin/env perl - -# -# fetch.pl -# -use warnings; -use File::stat; -use Months; - -# Call git on the products directory -my $git = "/usr/bin/env git --git-dir ../../ukf-products/.git"; - -$fn1 = 'aggregates/ukfederation-metadata.xml'; -$fn2 = 'aggregates/ukfederation-stats.html'; - -foreach $month (@months) { - print "Fetching $month..."; - - # Find the commit immediately prior to the start of that month. - my $instant = "$month-01T00:00:00Z"; - my $commit = `$git rev-list -n 1 --before=$instant master`; - chomp $commit; - print "$commit"; - - my $dest1 = "cache/$month.xml"; - if (!-e $dest1) { - system("$git show $commit:$fn1 >$dest1"); - } - - my $dest2 = "cache/$month.html"; - if (!-e $dest2) { - system("$git show $commit:$fn2 >$dest2"); - } - - print "\n"; -} diff --git a/charting/just_ours.xsl b/charting/just_ours.xsl deleted file mode 100644 index 3324d60f..00000000 --- a/charting/just_ours.xsl +++ /dev/null @@ -1,44 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/charting/mdui.pl b/charting/mdui.pl deleted file mode 100755 index 2651703b..00000000 --- a/charting/mdui.pl +++ /dev/null @@ -1,36 +0,0 @@ -#!/usr/bin/env perl - -# -# mdui.pl -# -use warnings; -use lib '.'; -use Months; - -# Parse command line arguments -use Getopt::Long; -my $allMonths; -my $oneYear; -GetOptions('all' => \$allMonths, 'year' => \$oneYear); - -# By default, only show results for the most recent month -if ($allMonths) { - # leave table intact -} elsif ($oneYear) { - # reduce months table to just the last 12 entries - @months = @months[-12..-1]; -} else { - # reduce months table to one element - @months = @months[-1..-1]; -} - -# ingest files -foreach $month (@months) { - print "Processing $month\n"; - - my $command = "xsltproc statistics_mdui.xsl cache/$month.xml"; - # print "command is $command\n"; - system($command); # || print "ignoring claimed failure in sub command\n"; - # print "xsltproc run on $fn\n"; - print "\n"; -} diff --git a/charting/requestedattribute.pl b/charting/requestedattribute.pl deleted file mode 100755 index 06ba6ce6..00000000 --- a/charting/requestedattribute.pl +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env perl - -# -# requestedattribute.pl -# -# Extracts statistics about SPs with RequestedAttribute elements from the published metadata. -# -use warnings; -use lib "."; -use Months; - -# Parse command line arguments -use Getopt::Long; -my $allMonths; -my $oneYear; -GetOptions('all' => \$allMonths, 'year' => \$oneYear); - -# By default, only show results for the most recent month -if ($allMonths) { - # leave table intact -} elsif ($oneYear) { - # reduce months table to just the last 12 entries - @months = @months[-12..-1]; -} else { - # reduce months table to one element - @months = @months[-1..-1]; -} - -# print header. must be kept in sync with what comes out of requestedattribute.xsl -print "# month, number of SPs, number with AttributeConsumingService, percent with ACS\n"; - -# ingest files -foreach $month (@months) { - my $fn = "cache/$month.xml"; - open(TXT, "xsltproc requestedattribute.xsl $fn|") || die "could not open input file"; - ($sps, $acs) = split /\t/, ; - chomp $acs; - $proportion = 0; - if ( $sps > 0 ) { $proportion = int (100 * $acs / $sps) ; } - print "${month}\t${sps}\t${acs}\t${proportion}\n"; - close TXT; -} - diff --git a/charting/requestedattribute.xsl b/charting/requestedattribute.xsl deleted file mode 100644 index c266f62a..00000000 --- a/charting/requestedattribute.xsl +++ /dev/null @@ -1,37 +0,0 @@ - - - - - - - - - - - - - - - - - - - diff --git a/charting/saml2.pl b/charting/saml2.pl deleted file mode 100755 index 0542f210..00000000 --- a/charting/saml2.pl +++ /dev/null @@ -1,68 +0,0 @@ -#!/usr/bin/env perl - -# -# saml2.pl -# -# Extracts statistics about SAML 2 adoption from the published metadata. -# -use warnings; -use lib "."; -use Months; - -# Parse command line arguments -use Getopt::Long; -my $allMonths; -GetOptions('all' => \$allMonths); - -# By default, only show results for the most recent month -if (!$allMonths) { - # reduce months table to one element - my $oneMonth = pop @months; - @months = ( $oneMonth ); -} - -# ingest files -foreach $month (@months) { - my $fn = "cache/$month.xml"; - open(TXT, "xsltproc saml2.xsl $fn|") || die "could not open input file"; - $_ = ; - chop; - # print "$month: $_\n"; - my ($entities, $idps, $sps, $saml2total, $saml2idp, $saml2sp) = split; - if ($entities == 0) { - # print "skipping $month: $_\n"; - next; - } - my $mPrefix = $allMonths ? "$month: " : ''; - my $oRatio = $saml2total/$entities; - push @overallRatio, "$mPrefix$oRatio"; - my $iRatio = $saml2idp/$idps; - push @idpRatio, "$mPrefix$iRatio"; - my $sRatio = $saml2sp/$sps; - push @spRatio, "$mPrefix$sRatio"; - my $p = ($saml2idp/$idps)*($saml2sp/$sps); - push @product, "$mPrefix$p"; - close TXT; -} - -print "idp\n"; -foreach $ratio (@idpRatio) { - print "$ratio\n"; -} - -print "sp\n"; -foreach $ratio (@spRatio) { - print "$ratio\n"; -} - -print "overall\n"; -foreach $ratio (@overallRatio) { - print "$ratio\n"; -} - -print "product\n"; -foreach $ratio (@product) { - print "$ratio\n"; -} - -1; diff --git a/charting/saml2.xsl b/charting/saml2.xsl deleted file mode 100644 index 04dd4455..00000000 --- a/charting/saml2.xsl +++ /dev/null @@ -1,52 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/charting/scopes.pl b/charting/scopes.pl deleted file mode 100755 index 8a8c6692..00000000 --- a/charting/scopes.pl +++ /dev/null @@ -1,50 +0,0 @@ -#!/usr/bin/env perl - -# -# scopes.pl -# -# Extracts statistics about number of scopes from the published metadata. -# -use warnings; -use lib "."; -use Months; - -# Parse command line arguments -use Getopt::Long; -my $allMonths; -my $oneYear; -GetOptions('all' => \$allMonths, 'year' => \$oneYear); - -# By default, only show results for the most recent month -if ($allMonths) { - # leave table intact -} elsif ($oneYear) { - # reduce months table to just the last 12 entries - @months = @months[-12..-1]; -} else { - # reduce months table to one element - @months = @months[-1..-1]; -} - -# ingest files -foreach $month (@months) { - my $fn = "cache/$month.xml"; - my %scopes; - open(TXT, "xsltproc scopes.xsl $fn|") || die "could not open input file"; - while () { - chop; - my $scope = $_; - $scopes{$scope} = 1; - } - my $prefix = scalar(@months) == 1 ? '' : "$month: "; - my $c = scalar(keys(%scopes)); - push @count, "$prefix$c"; - close TXT; -} - -print "count\n"; -foreach $n (@count) { - print "$n\n"; -} - -1; diff --git a/charting/scopes.xsl b/charting/scopes.xsl deleted file mode 100644 index 76e808c2..00000000 --- a/charting/scopes.xsl +++ /dev/null @@ -1,46 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/charting/sizes.pl b/charting/sizes.pl deleted file mode 100755 index 7f9fa924..00000000 --- a/charting/sizes.pl +++ /dev/null @@ -1,104 +0,0 @@ -#!/usr/bin/env perl - -# -# sizes.pl -# -use warnings; -use lib "."; -use File::stat; -use Months; - -# Parse command line arguments -use Getopt::Long; -my $allMonths; -my $oneYear; -GetOptions('all' => \$allMonths, 'year' => \$oneYear); - -# By default, only show results for the most recent month -if ($allMonths) { - # leave table intact -} elsif ($oneYear) { - # reduce months table to just the last 12 entries - @months = @months[-12..-1]; -} else { - # reduce months table to one element - @months = @months[-1..-1]; -} - -# ingest files -foreach $month (@months) { - print "Processing $month\n"; - - # - # Process the archived file, representing all entities, - # including those imported from other federations. - # - my $fn = "cache/$month.xml"; - my $stat = stat($fn); - my $all_size = $stat->size; - my $all_count = int(`grep 'size; - my $our_count = int(`grep ' - - - - - - - - - - - - - Total entities - - - - - Identity providers - - - - - Service providers - - - - - - - - - - - - : - - - - - - - mdui:UIInfo - - - - - - mdui:Logo - - - - - - mdui:Description - - - - - - mdui:DisplayName - - - - - - mdui:Keywords - - - - - - mdui:InformationURL - - - - - - mdui:PrivacyStatementURL - - - - - - mdui:DiscoHints - - - - - - mdui:IPHint - - - - - - mdui:DomainHint - - - - - - mdui:GeolocationHint - - - - - - - - - - - - - - - - : - - ( - - ) - - - -