From f44bdb9111c3cb3187f7c60423e8082a82998eee Mon Sep 17 00:00:00 2001 From: Keith Hazelton Date: Thu, 1 Aug 2024 22:07:22 -0500 Subject: [PATCH] Update postgres-only-workbench.adoc --- docs/postgres-only-workbench.adoc | 430 ++++++++++++++++++++++++++++++ 1 file changed, 430 insertions(+) diff --git a/docs/postgres-only-workbench.adoc b/docs/postgres-only-workbench.adoc index 1709b5f..bce39d6 100644 --- a/docs/postgres-only-workbench.adoc +++ b/docs/postgres-only-workbench.adoc @@ -37,3 +37,433 @@ docker ps . You will see a kiosk-like interface with links to the TAP components and to other supporting services . COmanage, Grouper, midPoint and a Shib IdP are directly accessible +- - - + +== 1. Isolating references to mysql and postgres in the TAP Workbench + +=== In docker-compose.yml + +*- mysql/mariadb -* + +``` +200: sources: + build: ./sources/ + volumes: + - source_mysql:/var/lib/mysql + - source_data:/var/lib/mysqlmounted + +480: wordpress_server: + build: + context: ./wordpress_server/ + command: bash -c 'if [ ! -s /var/www/html/wp-config.php ]; then while ! nc -z wordpress_data 3306 ; do echo waiting for mysql on wordpress_data to start; sleep 3; done; + +511: wordpress_data: + build: ./wordpress_data/ + volumes: + - wordpress_data:/var/lib/mysql + +650: volumes: + source_data: + comanage_mysql: + source_mysql: + target_data: + wordpress_data: + mariadb-data: +``` +*- postgres -* + +``` +221: comanage_data: + build: ./comanage_data/ + environment: + POSTGRES_USER: registry_user + POSTGRES_PASSWORD: Password1 + POSTGRES_DB: registry + volumes: + - comanage_data:/var/lib/postgresql/data + +242: comanage_midpoint_data: + build: ./comanage_midpoint_data/ + environment: + POSTGRES_USER: comanage_midpoint_loader + POSTGRES_PASSWORD: Password1 + POSTGRES_DB: comanage_midpoint_loader + networks: + net: + aliases: + - comanage-midpoint-data + volumes: + - comanage_midpoint_data:/var/lib/postgresql/data + + 282: midpoint_data: + image: postgres:13-alpine + command: > + rm -f /var/lib/postgresql/data/postmaster.pid ; + docker-entrypoint.sh postgres + environment: + - POSTGRES_PASSWORD_FILE=/run/secrets/m_database_password.txt + - POSTGRES_USER=midpoint + - POSTGRES_INITDB_ARGS=--lc-collate=en_US.utf8 --lc-ctype=en_US.utf8 + secrets: + - m_database_password.txt + volumes: + - midpoint_data:/var/lib/postgresql/data + - db_init:/docker-entrypoint-initdb.d/ + - mp_pw:/opt/mp-pw + +324: midpoint_server: + build: + context: ./midpoint_server/ + environment: + - REPO_DATABASE_TYPE=postgresql + m_database_password.txt + - MP_SET_midpoint_repository_jdbcUrl=jdbc:postgresql://midpoint_data:5432/midpoint + +432: idp_ui_data: + image: postgres + environment: + POSTGRES_USER: shibui + POSTGRES_PASSWORD: secret + POSTGRES_DB: shibui + volumes: + - idpui_data:/var/lib/postgresql/data + +``` + +- - - + +== 2. Migrate MySQL databases to Postgres + + +*- MySQL in Dockerfiles -* + +===== ./sources/Dockerfile + +``` +FROM tier/mariadb:mariadb10 + +COPY container_files/seed-data/ /seed-data/ + +ENV MYSQL_DATABASE sis +ENV MYSQL_USER sis_user +ENV MYSQL_PASSWORD 49321420423 +ENV MYSQL_DATADIR /var/lib/mysqlmounted +ENV AFTER_FIRST_TIME_SQL /seed-data/persons-and-courses.sql + +[csprootuser@ip-172-31-53-134 Workbench]$ cat sources/Dockerfile +FROM tier/mariadb:mariadb10 + +COPY container_files/seed-data/ /seed-data/ + +ENV MYSQL_DATABASE sis +ENV MYSQL_USER sis_user +ENV MYSQL_PASSWORD 49321420423 +ENV MYSQL_DATADIR /var/lib/mysqlmounted +ENV AFTER_FIRST_TIME_SQL /seed-data/persons-and-courses.sql +``` + +- - - + +== Other relevant selections from Docker-compose.yml and the per-component Dockerfiles + +*- Running Workbench Containers -* +``` +[csprootuser@ip-172-31-53-134 ~]$ docker ps + +bcac3920984a workbench-comanage_cron "docker-comanage-cro…" 11 days ago Up 11 days (healthy) 80/tcp, 443/tcp workbench-comanage_cron-1 + +73d6df7dc964 workbench-midpoint_server "/usr/local/bin/star…" 11 days ago Up 11 days (healthy) 80/tcp, 0.0.0.0:10443->443/tcp, :::10443->443/tcp workbench-midpoint_server-1 + +a7f80a6c2eb4 workbench-idp_ui_api "/usr/bin/supervisor…" 11 days ago Up 11 days (healthy) 8443/tcp workbench-idp_ui_api-1 + +1cdf5c3baf5d workbench-grouper_ui "/usr/local/bin/entr…" 11 days ago Up 11 days (healthy) 80/tcp, 8080/tcp, 8443/tcp, 0.0.0.0:8443->443/tcp, :::8443->443/tcp + +bfc26f8e0c9d workbench-grouper_daemon "/usr/local/bin/entr…" 11 days ago Up 11 days (healthy) 80/tcp, 443/tcp, 8080/tcp, 8443/tcp + +a3594f0d5542 workbench-grouper_ws "/usr/local/bin/entr…" 11 days ago Up 11 days (healthy) 80/tcp, 8080/tcp, 8443/tcp, 0.0.0.0:9443->443/tcp, :::9443->443/tcp + +30d2bb6f3835 workbench-idp "/usr/bin/startup.sh" 11 days ago Up 11 days (healthy) 0.0.0.0:13443->443/tcp, :::13443->443/tcp + +d12c56e97b9a postgres:13-alpine "docker-entrypoint.s…" 11 days ago Up 11 days (healthy) 0.0.0.0:5432->5432/tcp, :::5432->5432/tcp workbench-midpoint_data-1 + +a8f06373fdff workbench-comanage "docker-supervisord-…" 11 days ago Up 11 days (healthy) 80/tcp, 8080/tcp, 0.0.0.0:11443->443/tcp, :::11443->443/tcp + +b0d5d8ac7193 workbench-wordpress_server "bash -c 'if [ ! -s …" 11 days ago Up 11 days (healthy) 0.0.0.0:80->80/tcp, :::80->80/tcp, 0.0.0.0:12443->443/tcp, :::12443->443/tcp + +ff75fa2311c7 workbench-directory "/bin/sh -c 'rm -rf …" 11 days ago Up 11 days (healthy) 443/tcp, 0.0.0.0:1389->389/tcp, :::1389->389/tcp + +f949dd8a2791 workbench-idp_ui "/usr/bin/supervisor…" 11 days ago Up 11 days (healthy) 0.0.0.0:8080->8080/tcp, :::8080->8080/tcp, 8443/tcp + +342372c932b4 workbench-sources "/opt/bin/start.sh" 11 days ago Up 11 days (healthy) 0.0.0.0:13306->3306/tcp, :::13306->3306/tcp + +e3349853f115 workbench-webproxy "/usr/local/bin/star…" 11 days ago Up 11 days (healthy) 80/tcp, 0.0.0.0:443->443/tcp, :::443->443/tcp + +46483b5008e7 workbench-comanage_midpoint_data "docker-entrypoint.s…" 11 days ago Up 11 days (healthy) 0.0.0.0:35432->5432/tcp, :::35432->5432/tcp + +a5aac809e4bc workbench-grouper_data "/usr/local/bin/entr…" 11 days ago Up 11 days (healthy) 80/tcp, 443/tcp, 8080/tcp, 8443/tcp, 0.0.0.0:45432->5432/tcp, :::45432->5432/tcp + +55637013b5c7 workbench-mq "/usr/local/bin/demo…" 11 days ago Up 11 days (healthy) 4369/tcp, 5671/tcp, 0.0.0.0:5672->5672/tcp, :::5672->5672/tcp, 15671/tcp, 25672/tcp, 0.0.0.0:15672->15672/tcp, :::15672->15672/tcp + +00b1baa4939d workbench-wordpress_data "docker-entrypoint.s…" 11 days ago Up 11 days (healthy) 0.0.0.0:32773->3306/tcp, :::32773->3306/ + +be8bbd009ead workbench-ad "/bin/sh -c '/start.…" 11 days ago Up 11 days 0.0.0.0:53->53/tcp, ... + + +ba5775b6dbe7 workbench-comanage_data "docker-entrypoint.s…" 11 days ago Up 11 days (healthy) 0.0.0.0:25432->5432/tcp, :::25432->5432/ + +c54f3a94e07f postgres "docker-entrypoint.s…" 11 days ago Up 11 days (healthy) 0.0.0.0:15432->5432/tcp, :::15432->5432/ +``` + +*- Workbench directory -* + +``` +[csprootuser@ip-172-31-53-134 csp-tap]$ cd /csp-tap/InCommonTAP-Examples/Workbench +[csprootuser@ip-172-31-53-134 Workbench]$ ls -la +total 80 +drwxr-xr-x 23 csprootuser root 4096 Jul 10 17:26 . +drwxr-xr-x 6 csprootuser root 82 May 10 14:12 .. +drwxr-xr-x 3 csprootuser root 47 May 10 14:12 ad +drwxr-xr-x 3 csprootuser root 47 May 10 14:12 comanage +drwxr-xr-x 3 csprootuser root 47 May 10 14:12 comanage_cron +drwxr-xr-x 3 csprootuser root 47 May 10 14:12 comanage_data +drwxr-xr-x 3 csprootuser root 47 May 10 14:12 comanage_midpoint_data +drwxr-xr-x 6 csprootuser root 70 May 10 14:12 configs-and-secrets +drwxr-xr-x 3 csprootuser root 47 Jun 10 16:43 directory +-rw-r--r-- 1 csprootuser csprootuser 20547 Jul 10 16:26 docker-compose.yml +-rw-r--r-- 1 csprootuser root 20565 Jun 9 19:00 docker-compose.yml.old +-rw-r--r-- 1 csprootuser root 57 May 10 14:12 .env +drwxr-xr-x 2 csprootuser root 24 Jun 10 15:17 grouper_daemon +drwxr-xr-x 3 csprootuser root 47 May 10 14:12 grouper_data +drwxr-xr-x 3 csprootuser root 47 May 10 14:12 grouper_ui +drwxr-xr-x 3 csprootuser root 47 May 10 14:12 grouper_ws +drwxr-xr-x 4 csprootuser root 69 May 10 14:12 idp +drwxr-xr-x 3 csprootuser root 47 May 10 14:12 idp_ui +drwxr-xr-x 3 csprootuser root 47 May 10 14:12 idp_ui_api +drwxr-xr-x 3 csprootuser root 47 May 10 14:12 midpoint_server +drwxr-xr-x 3 csprootuser root 47 May 10 14:12 mq +-rw-r--r-- 1 csprootuser root 843 May 10 14:12 README.md +drwxr-xr-x 2 csprootuser root 232 May 10 14:21 scripts +drwxr-xr-x 3 csprootuser root 47 May 10 14:12 sources +-rw-rw-r-- 1 csprootuser csprootuser 17257 Jul 10 17:17 th.xml +drwxr-xr-x 3 csprootuser root 47 May 10 14:12 webproxy +drwxr-xr-x 2 csprootuser root 24 May 10 14:12 wordpress_data +drwxr-xr-x 4 csprootuser root 69 May 10 14:12 wordpress_server +``` +*- Workbench Dockerfiles and container_file directories for building containers -* + +``` +[csprootuser@ip-172-31-53-134 Workbench]$ tree -L 2 +. +├── ad +│   ├── container_files +│   └── Dockerfile +├── comanage +│   ├── container_files +│   └── Dockerfile +├── comanage_cron +│   ├── container_files +│   └── Dockerfile +├── comanage_data +│   ├── container_files +│   └── Dockerfile +├── comanage_midpoint_data +│   ├── container_files +│   └── Dockerfile +├── configs-and-secrets +│   ├── comanage +│   ├── grouper +│   ├── midpoint +│   └── wordpress +├── directory +│   ├── container_files +│   └── Dockerfile +├── docker-compose.yml +├── docker-compose.yml.old +├── grouper_daemon +│   └── Dockerfile +├── grouper_data +│   ├── container_files +│   └── Dockerfile +├── grouper_ui +│   ├── container_files +│   └── Dockerfile +├── grouper_ws +│   ├── container_files +│   └── Dockerfile +├── idp +│   ├── container_files +│   ├── Dockerfile +│   └── shibboleth-idp +├── idp_ui +│   ├── container_files +│   └── Dockerfile +├── idp_ui_api +│   ├── container_files +│   └── Dockerfile +├── midpoint_server +│   ├── container_files +│   └── Dockerfile +├── mq +│   ├── container_files +│   └── Dockerfile +├── README.md +├── scripts +│   ├── csp-cron +│   ├── gethealth2.py +│   ├── gethealth-output.txt +│   ├── gethealth.py +│   ├── refreshListener.php +│   ├── refreshListener.service +│   ├── refresh-this-instance.sh +│   ├── setupcron.sh +│   └── update-health-status.sh +├── sources +│   ├── container_files +│   └── Dockerfile +├── th.xml +├── webproxy +│   ├── container_files +│   └── Dockerfile +├── wordpress_data +│   └── Dockerfile +└── wordpress_server + ├── container_files + ├── Dockerfile + └── wordpress_data + + +/csp-tap/InCommonTAP-Examples/Workbench$ ls -la sources/container_files/seed-data +-rw-r--r-- 1 csprootuser root 55516 May 10 14:12 persons-and-courses.sql +``` + +== 2. Migrate MySQL databases to Postgres + +*- MySQL in Dockerfiles -* + +===== ./sources/Dockerfile + +``` +FROM tier/mariadb:mariadb10 + +COPY container_files/seed-data/ /seed-data/ + +ENV MYSQL_DATABASE sis +ENV MYSQL_USER sis_user +ENV MYSQL_PASSWORD 49321420423 +ENV MYSQL_DATADIR /var/lib/mysqlmounted +ENV AFTER_FIRST_TIME_SQL /seed-data/persons-and-courses.sql + +[csprootuser@ip-172-31-53-134 Workbench]$ cat sources/Dockerfile +FROM tier/mariadb:mariadb10 + +COPY container_files/seed-data/ /seed-data/ + +ENV MYSQL_DATABASE sis +ENV MYSQL_USER sis_user +ENV MYSQL_PASSWORD 49321420423 +ENV MYSQL_DATADIR /var/lib/mysqlmounted +ENV AFTER_FIRST_TIME_SQL /seed-data/persons-and-courses.sql +``` + +./wordpress_server/Dockerfile + +``` +cat wordpress_server/Dockerfile +FROM i2incommon/shibboleth_sp:3.4.1_06122023_rocky8_multiarch + +VOLUME /var/www/html + +COPY container_files/wordpress/sed.sh /root +COPY container_files/wordpress/config-shibb.sql /root +COPY container_files/wordpress/wp /root +COPY container_files/wordpress/config.yml /root/.wp-cli +COPY container_files/wordpress/wp-cli.yml /var/www/html +COPY container_files/shibboleth/* /etc/shibboleth/ +COPY container_files/system/setservername.sh /usr/local/bin/ +RUN chmod +x /root/wp +RUN dnf module enable -y php:7.4 +RUN yum update -y +RUN yum install -y php php-cli php-common php-gd php-curl php-json php-mysqlnd php-pdo php-zip php-mbstring libwebp mariadb wget postfix nc +RUN rpm -Uvh https://rpms.remirepo.net/enterprise/remi-release-8.rpm +RUN yum --enablerepo=remi,remi-test install -y gd3php gd3php-devel php74-php-sodium + +RUN echo 'date.timezone="UTC"' >> /etc/php.ini + +WORKDIR /var/www/html + +RUN chown -R apache:apache /var/www/html +COPY container_files/system/setservername.sh /usr/local/bin/ +RUN chmod 755 /usr/local/bin/setservername.sh #&& rm -f /etc/httpd/conf.d/ssl.conf + +#set hostname +ARG CSPHOSTNAME=localhost +ENV CSPHOSTNAME=$CSPHOSTNAME + +RUN /usr/local/bin/setservername.sh +RUN mkdir -p /run/php-fpm/ + +ENV LD_LIBRARY_PATH=/opt/shibboleth/lib64 +``` + +./wordpress_data/Dockerfile + +``` +$ cat wordpress_data/Dockerfile +from mariadb:latest +RUN apt-get update +RUN apt install curl -y +#RUN apt-get install wget gcc libmysql++-dev librabbitmq-dev pkg-config libbsd-dev -y +#ENV MYSQL_RANDOM_ROOT_PASSWORD=true +ENV MYSQL_ROOT_PASSWORD=54y6RxN7GfC7aes3 +ENV MYSQL_DATABASE=wordpress +ENV MYSQL_USER=wordpress +ENV MYSQL_PASSWORD=54y6RxN7GfC7aes3 +#WORKDIR /tmp +#RUN wget https://github.com/ssimicro/lib_mysqludf_amqp/releases/download/v2.0.0/lib_mysqludf_amqp-2.0.0.tar.gz +#RUN tar zxf lib_mysqludf_amqp-2.0.0.tar.gz +#WORKDIR /tmp/lib_mysqludf_amqp-2.0.0 +#RUN ./configure && make && make install #mysql -u root --password=54y6RxN7GfC7aes3 < installdb.sql +RUN cat /etc/resolv.conf +EXPOSE 3306 +``` + +*- Use grouper_data Dockerfile as template for migrating MySQL databases -* + +./grouper_data/Dockerfile + +``` +FROM i2incommon/grouper:4.12.0 + +LABEL author="tier-packaging@internet2.edu " + +RUN yum install -y epel-release \ + && yum update -y \ + && dnf module enable -y postgresql:12 \ + && dnf install -y postgresql-server \ + && yum clean all \ + && rm -rf /var/cache/yum + +COPY container_files/conf/ /opt/grouper/grouperWebapp/WEB-INF/classes/ +COPY container_files/bootstrap/ /tmp/ +COPY container_files/sql/createSQLuser.sql / +COPY container_files/sql/createDBforMP.sql / + +#setup DB +RUN chown -R postgres:postgres /var/lib/pgsql/ +RUN echo "password" > /db-user-pwd.txt +RUN sudo -u postgres initdb -D /var/lib/pgsql/data/ --username=postgres --pwfile=/db-user-pwd.txt + +#create grouper DB +RUN sudo -u postgres pg_ctl start -D /var/lib/pgsql/data/ \ + && psql -U postgres -f /createSQLuser.sql \ + && psql -U postgres -f /createDBforMP.sql \ + && /opt/grouper/grouperWebapp/WEB-INF/bin/gsh.sh -registry -check -runscript -noprompt \ + && /opt/grouper/grouperWebapp/WEB-INF/bin/gsh.sh /tmp/initialize.gsh \ + && /opt/grouper/grouperWebapp/WEB-INF/bin/gsh.sh /tmp/set-prov.gsh + +RUN echo "host all all 0.0.0.0/0 trust" >> /var/lib/pgsql/data/pg_hba.conf + +EXPOSE 5432 +``` + +