From 91875029d119ffe3694c90a735e0cd208e473710 Mon Sep 17 00:00:00 2001 From: Sean Porth Date: Wed, 7 Sep 2022 15:03:34 -0400 Subject: [PATCH] shibui-2378 move bulk-upload into repo --- bulk-upload/README.md | 43 +++ bulk-upload/docker-build/Dockerfile | 10 + bulk-upload/docker-build/README.txt | 1 + bulk-upload/docker-build/shibui-md-upload.pl | 337 +++++++++++++++++++ bulk-upload/run.sh | 15 + bulk-upload/upload.conf | 35 ++ 6 files changed, 441 insertions(+) create mode 100644 bulk-upload/README.md create mode 100644 bulk-upload/docker-build/Dockerfile create mode 100644 bulk-upload/docker-build/README.txt create mode 100755 bulk-upload/docker-build/shibui-md-upload.pl create mode 100755 bulk-upload/run.sh create mode 100644 bulk-upload/upload.conf diff --git a/bulk-upload/README.md b/bulk-upload/README.md new file mode 100644 index 000000000..c436394e8 --- /dev/null +++ b/bulk-upload/README.md @@ -0,0 +1,43 @@ +This script can be used to bulk upload metadata files into the Shibboleth IdP UI + +### requirements ### +The shibui must be configured without SAML authentication enabled. The API is currently only accessible using Basic auth. +If shibui is using SAML auth, you can temporarily disable it by setting pac4j-enabled: false in application.yml and restarting shibui. +After uploading you can re-enable SAML auth by setting pac4j-enabled: true in application.yml and restarting shibui. + +### usage ### +`./run.sh -e` + +Where is one of: + +path to a dir containing individual metadata files, aggregate files, or a mix of both +path to one metadata file or aggregate file. + +-e enable metadata source after upload + +### configuration ### +Configuration is done in upload.conf. This file must reside in the same dir as run.sh +At a minimum it must contain the shibui api info. It can also be used to add entity attributes to the metadata as it is uploaded (examples are in upload.conf). + +#a entity attribute configuration in upload.conf like: +attr[0][FriendlyName] = signAssertions +attr[0][Name] = http://shibboleth.net/ns/profiles/saml2/sso/browser/signAssertions +attr[0][type] = xsd:boolean +attr[0][Value] = false + +#turns into: +```xml + + false + +``` + +### docker image ### +run.sh will pull the shibui-bulk-upload image from dockerhub and run the script within +If you need to build the image yourself, `cd docker-build;docker build -t unicon/shibui-bulk-upload .` + +### running without docker ### +The docker-build/shibui-md-upload.pl script can be run manually. +Depending on OS you may need to install some additional non dist Perl modules. REST::Client , XML::LibXML , JSON , Config::File +`perl shibui-md-upload.pl -c -m -e (enable metadata source after upload)` + diff --git a/bulk-upload/docker-build/Dockerfile b/bulk-upload/docker-build/Dockerfile new file mode 100644 index 000000000..dd7384667 --- /dev/null +++ b/bulk-upload/docker-build/Dockerfile @@ -0,0 +1,10 @@ +FROM debian:stable-slim +MAINTAINER sporth@unicon.net + +RUN apt-get clean \ + && apt-get -y update \ + && apt-get install -y librest-client-perl libconfig-file-perl jq libxml-libxml-perl libjson-perl + +WORKDIR /opt +COPY shibui-md-upload.pl /opt/shibui-md-upload.pl +ENTRYPOINT ["/opt/shibui-md-upload.pl"] diff --git a/bulk-upload/docker-build/README.txt b/bulk-upload/docker-build/README.txt new file mode 100644 index 000000000..823f746c5 --- /dev/null +++ b/bulk-upload/docker-build/README.txt @@ -0,0 +1 @@ +docker build -t unicon/shibui-bulk-upload . diff --git a/bulk-upload/docker-build/shibui-md-upload.pl b/bulk-upload/docker-build/shibui-md-upload.pl new file mode 100755 index 000000000..3aae45aa8 --- /dev/null +++ b/bulk-upload/docker-build/shibui-md-upload.pl @@ -0,0 +1,337 @@ +#!/usr/bin/perl + +use strict; +use warnings; + +use Getopt::Std; +use Config::File; +use REST::Client; +use MIME::Base64; +use HTTP::Cookies; +use XML::LibXML; +use XML::LibXML::XPathContext; +use Encode; +use JSON; + +##process arguments +our ($opt_e,$opt_m,$opt_c); +getopts('em:c:'); +my $enable = ($opt_e) ? 'true' : 'false'; +my $md = ($opt_m) ? $opt_m : '/opt/metadata/'; +my $uc = ($opt_c) ? $opt_c : '/opt/conf/upload.conf'; +## + +#config file is required, will contain the api information and other optional settings for adding extensions to the MD +my $conf = Config::File::read_config_file($uc); + +##setup REST::Client +my $unpw = encode_base64($conf->{api_user} . ':' . $conf->{api_pass}); +my $cookies = HTTP::Cookies->new( {} ); +my $client = REST::Client->new(); +$client->getUseragent()->cookie_jar($cookies); + +if ($conf->{api_selfsigned} =~ m/true/) { + $client->getUseragent()->ssl_opts(verify_hostname => 0); + $client->getUseragent()->ssl_opts(SSL_verify_mode => 0); +} + +$client->addHeader('Accept', 'application/json'); +$client->addHeader('Authorization', "Basic $unpw"); +$client->setHost($conf->{api_host}); + +#create auth token +$client->HEAD('/'); +my $token = &get_cookie_value($cookies, 'XSRF-TOKEN'); +$client->addHeader('X-XSRF-TOKEN', $token); +## + +#for stats +my $i = 0; +my $j = 0; + +##load the MD +#check whether we are reading a dir or file +if (-d $md) { + opendir(my $DH, $md) or die "Can't open $md: $!"; + + while (readdir $DH) { + next if ($_ =~ m/^\.|\.\./); + my $file = $_; + + my $xpc = &load_xml("$md/$file"); + + #check if file is aggregate, or individual entity + my $root = $xpc->findnodes("/*[local-name()='EntitiesDescriptor']"); + + if ($root) { + print "\nprocessing aggregate file: $file ....\n"; + + my $entity = $xpc->findnodes("//*[local-name()='EntityDescriptor']"); + + foreach my $var ($entity->get_nodelist) { + my ($name,$entid,$xml) = &get_entity($var); + print "\nimporting entity $entid\n"; + + #check if config file indicates entity attributes are to be added and add them to the parsed xml as needed + my $attr = get_attr_xml($xpc,$xml); + $xml = $attr if ($attr); + + my $code = &call_api($name,$entid,$xml,$enable); + $i++ if ($code == 201); + $j++ if ($code == 409); + } + + } else { + print "\nprocessing MD file: $file ....\n"; + + open(my $fh,"$md/$file"); + read $fh, my $xml, -s $fh; + + my ($name,$entid) = &get_names($xpc); + + #check if config file indicates entity attributes are to be added and add them to the parsed xml as needed + my $attr = get_attr_xml($xpc,$xml); + $xml = $attr if ($attr); + + my $code = &call_api($name,$entid,$xml,$enable); + $i++ if ($code == 201); + $j++ if ($code == 409); + } + } + closedir $DH; + +#single file +} elsif (-f $md) { + my $xpc = &load_xml("$md"); + + #check if file is aggregate, or individual entity + my $root = $xpc->findnodes("/*[local-name()='EntitiesDescriptor']"); + + if ($root) { + print "\nprocessing aggregate file: $md ....\n"; + + my $entity = $xpc->findnodes("//*[local-name()='EntityDescriptor']"); + + foreach my $var ($entity->get_nodelist) { + my ($name,$entid,$xml) = &get_entity($var); + print "\nimporting entity $entid\n"; + + #check if config file indicates entity attributes are to be added and add them to the parsed xml as needed + my $attr = get_attr_xml($xpc,$xml); + $xml = $attr if ($attr); + + my $code = &call_api($name,$entid,$xml,$enable); + $i++ if ($code == 201); + $j++ if ($code == 409); + } + + } else { + print "\nprocessing MD file: $md ....\n"; + + open(my $fh,$md); + read $fh, my $xml, -s $fh; + + my ($name,$entid) = &get_names($xpc); + + #check if config file indicates entity attributes are to be added and add them to the parsed xml as needed + my $attr = get_attr_xml($xpc,$xml); + $xml = $attr if ($attr); + + my $code = &call_api($name,$entid,$xml,$enable); + $i++ if ($code == 201); + $j++ if ($code == 409); + } + +} else { + print "$md can not be found\n"; + exit 1; +} + +print "\nmetadata uploaded: $i\nduplicate: $j\n"; + +sub load_xml { + my $file = shift; + + my $dom = XML::LibXML->load_xml(location => "$file"); + my $xpc = XML::LibXML::XPathContext->new(); + $xpc->registerNs('md', 'urn:oasis:names:tc:SAML:2.0:metadata'); + $xpc->registerNs('mdui', 'urn:oasis:names:tc:SAML:metadata:ui'); + $xpc->setContextNode($dom); + return $xpc; +} + +sub get_cookie_value { + my $cookies = $_[0]; + my $name = $_[1]; + my $result = 0; + + $cookies->scan(sub + { + if ($_[1] eq $name) + { + $result = $_[2]; + }; + }); + + return $result; +} + +sub get_names { + my $node = shift; + my $name; + + my $entid = $node->findnodes("//*[local-name()='EntityDescriptor']/\@entityID"); + my $orgname = $node->findnodes("//*[local-name()='Organization']/*[local-name()='OrganizationDisplayName']/text()"); + my $uiname = $node->findnodes("//*[local-name()='UIInfo']/*[local-name()='DisplayName']/text()"); + $name = ($orgname) ? $orgname : $entid; + $name = ($uiname) ? $uiname : $name; + + return ($name,$entid); +} + + +sub call_api { + my $name = shift; + my $entid = shift; + my $xml = shift; + my $enable = shift; + my ($params,$code,$result); + + my $utf8 = encode_utf8($xml); + + $params = "?spName=$name"; + $params .= "&enableService=true" if ($enable); + + $client->addHeader('Content-Type', "application/xml; charset='utf8'"); + $client->POST("/api/EntityDescriptor$params",$utf8); + + $code = $client->responseCode(); + $result = $client->responseContent(); + + if ($code == 201) { + + if ($enable =~ m/true/) { + + my $res = JSON->new->decode($result); + my $id = $res->{id}; + + $client->PATCH("/api/activate/entityDescriptor/$id/enable"); + + my $ecode = $client->responseCode(); + my $eresult = $client->responseContent(); + + if ($ecode == 200 || $ecode == 201) { + print "$ecode: entity $name uploaded sucessfully and enabled\n"; + } else { + print "$ecode: entity $name uploaded sucessfully but enabling failed:\n"; + open(my $pipe, '|-', "jq ."); + print $pipe $eresult; + } + + } else { + print "$code: entity $name uploaded sucessfully\n"; + } + } elsif ($code == 409) { + print "$code: entity $name already exists\n"; + } elsif ($code == 500) { + print "$code: $result\n"; + } else { + open(my $pipe, '|-', "jq ."); + print $pipe $result; + } + + return $code; +} + +sub get_entity { + my $node = shift; + + my $entid = $node->findnodes('./@entityID'); + my $orgname = $node->findnodes("./*[local-name()='Organization']/*[local-name()='OrganizationDisplayName']/text()"); + my $uiname = $node->findnodes("./*[local-name()='SPSSODescriptor']/*[local-name()='Extensions']/*[local-name()='UIInfo']/*[local-name()='DisplayName']/text()"); + my $name = ($orgname) ? $orgname : $entid; + $name = ($uiname) ? $uiname : $name; + + #have to add the NS from EntityDescriptor so each parsed xml fragment is valid + my $ns = qq(xmlns="urn:oasis:names:tc:SAML:2.0:metadata" xmlns:alg="urn:oasis:names:tc:SAML:metadata:algsupport" xmlns:mdrpi="urn:oasis:names:tc:SAML:metadata:rpi" xmlns:mdattr="urn:oasis:names:tc:SAML:metadata:attribute" xmlns:saml="urn:oasis:names:tc:SAML:2.0:assertion" xmlns:ds="http://www.w3.org/2000/09/xmldsig#" xmlns:mdui="urn:oasis:names:tc:SAML:metadata:ui" xmlns:remd="http://refeds.org/metadata" xmlns:idpdisc="urn:oasis:names:tc:SAML:profiles:SSO:idp-discovery-protocol" xmlns:shibmd="urn:mace:shibboleth:metadata:1.0"); + + $node =~ s/\\n{attr}; + my ($nxml,$vxml); + + if ($attr) { + my $n = keys %$attr; + + #if md:Extensions + my $ext = $xpc->findnodes("//*[local-name()='Extensions']"); + if ($ext) { + my $ea = $xpc->findnodes("//*[local-name()='EntityAttributes']"); + if ($ea) { + $nxml .= &create_attrs($attr,$n); + $xml =~ s/\<\/mdattr\:EntityAttributes\>/$nxml<\/mdattr:EntityAttributes>/; + } else { + $nxml = qq(); + $nxml .= &create_attrs($attr,$n); + $nxml .= qq(); + + $xml =~ s/\<\/md\:Extensions\>/$nxml<\/md:Extensions>/; + } + } else { + $nxml = qq(); + $nxml .= qq(); + $nxml .= &create_attrs($attr,$n); + $nxml .= qq(); + $nxml .= qq(); + + $xml =~ s/\<\/md\:EntityDescriptor\>/$nxml<\/md:EntityDescriptor>/; + } + + return $xml; + } +return 0; +} + +sub create_attrs { + my $attr = shift; + my $n = shift; + my ($nxml,$vxml); + + for(my $i = 0; $i < $n; $i++){ + my @val; + my $afname = $attr->{$i}{FriendlyName}; + my $aname = $attr->{$i}{Name}; + my $atype = $attr->{$i}{type}; + my $avalue = $attr->{$i}{Value}; + + if (ref $avalue eq ref {}) { + my $n = keys %$avalue; + for(my $j = 0; $j < $n; $j++){ + push(@val,$attr->{$i}{Value}{$j}); + } + } else { + push(@val,$avalue); + } + + undef $vxml; + foreach my $var (@val) { + $vxml .= qq($var\n); + } + $vxml =~ s/\n$//; + + $nxml .= qq( + + $vxml + ); + + } + $nxml =~ s/\<\/saml2\:Attribute\>$/<\/saml2:Attribute>\n/; + + return $nxml; +} diff --git a/bulk-upload/run.sh b/bulk-upload/run.sh new file mode 100755 index 000000000..86c128c6f --- /dev/null +++ b/bulk-upload/run.sh @@ -0,0 +1,15 @@ +#!/bin/sh + +MD=$1 +LAST=$(basename "$MD") + +if [ -z "$1" ];then +echo "usage: ./run.sh or -e (to enable metadata as it is uploaded)"; +exit 1; +fi + +if [ -z "${1%%/*}" ];then + docker run --network=host --mount type=bind,source=$MD,target=/opt/$LAST --mount type=bind,source=$PWD/upload.conf,target=/opt/conf/upload.conf -t unicon/shibui-bulk-upload -m /opt/$LAST $2 +else + docker run --network=host --mount type=bind,source=$PWD/$MD,target=/opt/$LAST --mount type=bind,source=$PWD/upload.conf,target=/opt/conf/upload.conf -t unicon/shibui-bulk-upload -m /opt/$LAST $2 +fi diff --git a/bulk-upload/upload.conf b/bulk-upload/upload.conf new file mode 100644 index 000000000..b0a82d6e5 --- /dev/null +++ b/bulk-upload/upload.conf @@ -0,0 +1,35 @@ + +##shibui API info +#required + +api_user=root +api_pass=letmein7 +api_host=https://shibui.unicon.local + +#if the API is running with self signed certs +api_selfsigned=true + +## + +##EntityAttributes +#optional +#this configuration will transform these entity attribute values into xml and add them to the parsed xml as it us uploaded +#to add more attributes copy the first block and increment the index + +#single value example + +#attr[0][FriendlyName] = signAssertions +#attr[0][Name] = http://shibboleth.net/ns/profiles/saml2/sso/browser/signAssertions +#attr[0][type] = xsd:boolean +#attr[0][Value] = false + +#multi value example + +#attr[1][FriendlyName] = nameIDFormatPrecedence +#attr[1][Name] = http://shibboleth.net/ns/profiles/nameIDFormatPrecedence +#attr[1][type] = xsd:string +#attr[1][Value][0] = urn:oasis:names:tc:SAML:1.1:nameid-format:emailAddress +#attr[1][Value][1] = urn:oasis:names:tc:SAML:2.0:nameid-format:transient +#attr[1][Value][2] = urn:oasis:names:tc:SAML:2.0:nameid-format:persistent + +## \ No newline at end of file