From 32a3a4308c8f43628845a447e0da7a50801c7b99 Mon Sep 17 00:00:00 2001 From: Timothy Middelkoop Date: Fri, 14 Jan 2022 22:42:36 +0000 Subject: [PATCH] GCP: Cleanup visuals and outputs for cli storage. --- content/GCP/05_cli_storage.ipynb | 246 ++++++++++++++++--------------- 1 file changed, 127 insertions(+), 119 deletions(-) diff --git a/content/GCP/05_cli_storage.ipynb b/content/GCP/05_cli_storage.ipynb index 03c6b92..ea543f2 100644 --- a/content/GCP/05_cli_storage.ipynb +++ b/content/GCP/05_cli_storage.ipynb @@ -11,9 +11,9 @@ "```{admonition} Overview\n", ":class: tip\n", "\n", - "**Teaching:**\n", + "**Teaching:** 20 min\n", "\n", - "**Exercises:**\n", + "**Exercises:** 2 min\n", "\n", "**Questions:**\n", "* How do I store data in a Bucket?\n", @@ -53,8 +53,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Your active configuration is: [cloudshell-31923]\n", - "student31@class.internet2.edu\n" + "998995103712-compute@developer.gserviceaccount.com\n" ] } ], @@ -72,8 +71,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Your active configuration is: [cloudshell-31923]\n", - "just-armor-301114\n" + "essentials-learner\n" ] } ], @@ -91,7 +89,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "just-armor-301114\n" + "essentials-learner\n" ] } ], @@ -120,7 +118,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 4, "id": "255eb914-a06a-4bfd-adbb-7ca2f94678e7", "metadata": {}, "outputs": [ @@ -128,7 +126,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "essentials-learner-2021-12-16\n" + "essentials-learner-2022-01-14\n" ] } ], @@ -139,7 +137,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 5, "id": "b0661b4c-b56b-4d6d-ba3c-e1c5f38699c6", "metadata": {}, "outputs": [ @@ -147,7 +145,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Creating gs://essentials-learner-2021-12-16/...\n" + "Creating gs://essentials-learner-2022-01-14/...\n" ] } ], @@ -185,7 +183,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "gs://essentials-student31-2021-10-26/\n" + "gs://essentials-learner-2022-01-14/\n" ] } ], @@ -219,7 +217,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "gs://essentials-student31-2021-10-26/ :\n", + "gs://essentials-learner-2022-01-14/ :\n", "\tStorage class:\t\t\tSTANDARD\n", "\tLocation type:\t\t\tmulti-region\n", "\tLocation constraint:\t\tUS\n", @@ -231,34 +229,34 @@ "\tRequester Pays enabled:\t\tNone\n", "\tLabels:\t\t\t\tNone\n", "\tDefault KMS key:\t\tNone\n", - "\tTime created:\t\t\tTue, 26 Oct 2021 20:47:06 GMT\n", - "\tTime updated:\t\t\tTue, 26 Oct 2021 20:47:06 GMT\n", + "\tTime created:\t\t\tFri, 14 Jan 2022 22:25:26 GMT\n", + "\tTime updated:\t\t\tFri, 14 Jan 2022 22:25:26 GMT\n", "\tMetageneration:\t\t\t1\n", "\tBucket Policy Only enabled:\tFalse\n", - "\tPublic access prevention:\tunspecified\n", + "\tPublic access prevention:\tinherited\n", "\tRPO:\t\t\t\tDEFAULT\n", "\tACL:\t\t\t\t\n", "\t [\n", "\t {\n", - "\t \"entity\": \"project-owners-1002111293252\",\n", + "\t \"entity\": \"project-owners-998995103712\",\n", "\t \"projectTeam\": {\n", - "\t \"projectNumber\": \"1002111293252\",\n", + "\t \"projectNumber\": \"998995103712\",\n", "\t \"team\": \"owners\"\n", "\t },\n", "\t \"role\": \"OWNER\"\n", "\t },\n", "\t {\n", - "\t \"entity\": \"project-editors-1002111293252\",\n", + "\t \"entity\": \"project-editors-998995103712\",\n", "\t \"projectTeam\": {\n", - "\t \"projectNumber\": \"1002111293252\",\n", + "\t \"projectNumber\": \"998995103712\",\n", "\t \"team\": \"editors\"\n", "\t },\n", "\t \"role\": \"OWNER\"\n", "\t },\n", "\t {\n", - "\t \"entity\": \"project-viewers-1002111293252\",\n", + "\t \"entity\": \"project-viewers-998995103712\",\n", "\t \"projectTeam\": {\n", - "\t \"projectNumber\": \"1002111293252\",\n", + "\t \"projectNumber\": \"998995103712\",\n", "\t \"team\": \"viewers\"\n", "\t },\n", "\t \"role\": \"READER\"\n", @@ -267,25 +265,25 @@ "\tDefault ACL:\t\t\t\n", "\t [\n", "\t {\n", - "\t \"entity\": \"project-owners-1002111293252\",\n", + "\t \"entity\": \"project-owners-998995103712\",\n", "\t \"projectTeam\": {\n", - "\t \"projectNumber\": \"1002111293252\",\n", + "\t \"projectNumber\": \"998995103712\",\n", "\t \"team\": \"owners\"\n", "\t },\n", "\t \"role\": \"OWNER\"\n", "\t },\n", "\t {\n", - "\t \"entity\": \"project-editors-1002111293252\",\n", + "\t \"entity\": \"project-editors-998995103712\",\n", "\t \"projectTeam\": {\n", - "\t \"projectNumber\": \"1002111293252\",\n", + "\t \"projectNumber\": \"998995103712\",\n", "\t \"team\": \"editors\"\n", "\t },\n", "\t \"role\": \"OWNER\"\n", "\t },\n", "\t {\n", - "\t \"entity\": \"project-viewers-1002111293252\",\n", + "\t \"entity\": \"project-viewers-998995103712\",\n", "\t \"projectTeam\": {\n", - "\t \"projectNumber\": \"1002111293252\",\n", + "\t \"projectNumber\": \"998995103712\",\n", "\t \"team\": \"viewers\"\n", "\t },\n", "\t \"role\": \"READER\"\n", @@ -333,7 +331,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 11, "id": "bb8e7270-2f64-4fbd-b312-b18a7960700f", "metadata": { "scrolled": true, @@ -345,65 +343,55 @@ "output_type": "stream", "text": [ "---\n", - "insertId: -z8eaicef61t3\n", - "logName: projects/just-armor-301114/logs/cloudaudit.googleapis.com%2Factivity\n", + "insertId: 1dyc72ccp1b\n", + "logName: projects/essentials-learner/logs/cloudaudit.googleapis.com%2Factivity\n", "protoPayload:\n", " '@type': type.googleapis.com/google.cloud.audit.AuditLog\n", " authenticationInfo:\n", - " principalEmail: student31@class.internet2.edu\n", + " principalEmail: 998995103712-compute@developer.gserviceaccount.com\n", + " principalSubject: serviceAccount:998995103712-compute@developer.gserviceaccount.com\n", + " serviceAccountDelegationInfo:\n", + " - firstPartyPrincipal:\n", + " principalEmail: service-998995103712@compute-system.iam.gserviceaccount.com\n", " authorizationInfo:\n", " - granted: true\n", - " permission: storage.buckets.create\n", - " resource: projects/_/buckets/essentials-student31-2021-10-26\n", + " permission: serviceusage.services.enable\n", + " resource: projectnumbers/998995103712\n", + " resourceAttributes: {}\n", + " - granted: true\n", + " permission: serviceusage.services.enable\n", + " resource: projectnumbers/998995103712\n", + " resourceAttributes: {}\n", + " - granted: true\n", + " permission: serviceusage.operations.get\n", + " resource: projectnumbers/998995103712/operations/-\n", + " resourceAttributes: {}\n", + " - granted: true\n", + " permission: serviceusage.services.get\n", + " resource: projectnumbers/998995103712/services/-\n", " resourceAttributes: {}\n", - " methodName: storage.buckets.create\n", - " request:\n", - " defaultObjectAcl:\n", - " '@type': type.googleapis.com/google.iam.v1.Policy\n", - " bindings:\n", - " - members:\n", - " - projectViewer:just-armor-301114\n", - " role: roles/storage.legacyObjectReader\n", - " - members:\n", - " - projectOwner:just-armor-301114\n", - " - projectEditor:just-armor-301114\n", - " role: roles/storage.legacyObjectOwner\n", + " methodName: google.longrunning.Operations.GetOperation\n", " requestMetadata:\n", - " callerIp: 35.239.199.87\n", - " callerSuppliedUserAgent: apitools Python/3.7.3 gsutil/5.4 (linux) analytics/disabled\n", - " interactive/True command/mb google-cloud-sdk/361.0.0,gzip(gfe)\n", + " callerIp: 34.94.169.98\n", + " callerNetwork: //compute.googleapis.com/projects/essentials-learner/global/networks/__unknown__\n", + " callerSuppliedUserAgent: google-cloud-sdk gcloud/366.0.0 command/gcloud.logging.read\n", + " invocation-id/90374af093e24c7d941f525d223d83c7 environment/GCE environment-version/None\n", + " interactive/True from-script/False python/3.7.3 term/xterm-256color (Linux 4.19.0-18-cloud-amd64),gzip(gfe)\n", " destinationAttributes: {}\n", " requestAttributes:\n", " auth: {}\n", - " time: '2021-10-26T20:47:05.790066575Z'\n", - " resourceLocation:\n", - " currentLocations:\n", - " - us\n", - " resourceName: projects/_/buckets/essentials-student31-2021-10-26\n", - " serviceData:\n", - " '@type': type.googleapis.com/google.iam.v1.logging.AuditData\n", - " policyDelta:\n", - " bindingDeltas:\n", - " - action: ADD\n", - " member: projectEditor:just-armor-301114\n", - " role: roles/storage.legacyBucketOwner\n", - " - action: ADD\n", - " member: projectOwner:just-armor-301114\n", - " role: roles/storage.legacyBucketOwner\n", - " - action: ADD\n", - " member: projectViewer:just-armor-301114\n", - " role: roles/storage.legacyBucketReader\n", - " serviceName: storage.googleapis.com\n", - " status: {}\n", - "receiveTimestamp: '2021-10-26T20:47:06.793562424Z'\n", + " time: '2022-01-14T22:34:23.880895Z'\n", + " resourceName: projects/998995103712/operations/acf.p2-998995103712-a617aa34-1a77-414c-bba6-7f6c931297db\n", + " serviceName: serviceusage.googleapis.com\n", + "receiveTimestamp: '2022-01-14T22:34:23.996006156Z'\n", "resource:\n", " labels:\n", - " bucket_name: essentials-student31-2021-10-26\n", - " location: us\n", - " project_id: just-armor-301114\n", - " type: gcs_bucket\n", + " method: google.longrunning.Operations.GetOperation\n", + " project_id: essentials-learner\n", + " service: serviceusage.googleapis.com\n", + " type: audited_resource\n", "severity: NOTICE\n", - "timestamp: '2021-10-26T20:47:05.785456012Z'\n" + "timestamp: '2022-01-14T22:34:23.839589Z'\n" ] } ], @@ -411,6 +399,23 @@ "gcloud logging read --limit 1" ] }, + { + "cell_type": "markdown", + "id": "e9e48571-80fa-40cc-838f-ec5ea3000bd7", + "metadata": {}, + "source": [ + "````{admonition} Note\n", + ":class: note\n", + "\n", + "You will probably need to enable the logging api to continue, press `y` to accept as shown below:\n", + "```\n", + "API [logging.googleapis.com] not enabled on project [998995103712]. Would you \n", + "like to enable and retry (this will take a few minutes)? (y/N)?\n", + "```\n", + "\n", + "````" + ] + }, { "cell_type": "markdown", "id": "fcdf7049-620f-4721-bc62-fa8f7fe97dcb", @@ -431,7 +436,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 13, "id": "f9a90752-871f-4156-b122-4c46fd4bfa43", "metadata": {}, "outputs": [], @@ -449,7 +454,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 14, "id": "bc0fb778-cc2e-4af4-8aa8-ca90d6b4f877", "metadata": {}, "outputs": [ @@ -475,7 +480,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 15, "id": "6384974b-d422-4ac1-90b7-cf9feec9aa8c", "metadata": {}, "outputs": [ @@ -495,7 +500,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 16, "id": "49589119-56bb-4c7b-b0a6-36e8f1133056", "metadata": {}, "outputs": [ @@ -503,7 +508,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "gs://essentials-student31-2021-10-26/1\n" + "gs://essentials-learner-2022-01-14/1\n" ] } ], @@ -529,7 +534,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 17, "id": "442cc51f-8856-41d4-a3a6-072c8569beef", "metadata": {}, "outputs": [ @@ -547,7 +552,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 18, "id": "4e3c0de9-8a23-428a-9689-0cf896228afe", "metadata": {}, "outputs": [ @@ -573,7 +578,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 19, "id": "13f94b36-7dea-46b9-9e8b-baa8b4e59de6", "metadata": {}, "outputs": [ @@ -581,7 +586,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Copying gs://essentials-student31-2021-10-26/1...\n", + "Copying gs://essentials-learner-2022-01-14/1...\n", "/ [1 files][ 9.0 B/ 9.0 B] \n", "Operation completed over 1 objects/9.0 B. \n" ] @@ -593,7 +598,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 20, "id": "3982f21c-b827-4772-aa8c-3ac67e478b0d", "metadata": {}, "outputs": [ @@ -619,7 +624,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 21, "id": "6b57a3fc-94ea-4c25-b7ac-2744d7b28852", "metadata": {}, "outputs": [ @@ -645,7 +650,7 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 22, "id": "5a7fc453-fca7-4ab4-8f0a-cb4de53bc992", "metadata": {}, "outputs": [ @@ -673,7 +678,7 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 23, "id": "1be58d86-5de7-420c-a3b3-0b82839ac90f", "metadata": {}, "outputs": [ @@ -681,8 +686,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "gs://essentials-student31-2021-10-26/1\n", - "gs://essentials-student31-2021-10-26/2\n" + "gs://essentials-learner-2022-01-14/1\n", + "gs://essentials-learner-2022-01-14/2\n" ] } ], @@ -695,12 +700,15 @@ "id": "1427e837-f65f-4d35-8088-6a82677b5ed6", "metadata": {}, "source": [ - "Exercise: display the date in the bucket." + "```{admonition} Exercise\n", + "\n", + "* Display the date in the bucket.\n", + "```" ] }, { "cell_type": "code", - "execution_count": 21, + "execution_count": 24, "id": "e3d0585b-4d81-4429-ad90-b3ff51eec13d", "metadata": {}, "outputs": [ @@ -708,7 +716,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Tue 26 Oct 2021 07:49:59 PM UTC\n" + "Fri Jan 14 22:35:35 UTC 2022\n" ] } ], @@ -726,7 +734,7 @@ }, { "cell_type": "code", - "execution_count": 23, + "execution_count": 25, "id": "9ad460d9-7a4f-4e3f-befb-2bbb10c51993", "metadata": {}, "outputs": [ @@ -734,7 +742,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Removing gs://essentials-student31-2021-10-26/1...\n", + "Removing gs://essentials-learner-2022-01-14/1...\n", "/ [1 objects] \n", "Operation completed over 1 objects. \n" ] @@ -746,7 +754,7 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": 26, "id": "8fd782f2-700a-4862-970b-1b4751009d5f", "metadata": {}, "outputs": [ @@ -754,7 +762,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "gs://essentials-student31-2021-10-26/2\n" + "gs://essentials-learner-2022-01-14/2\n" ] } ], @@ -772,7 +780,7 @@ }, { "cell_type": "code", - "execution_count": 25, + "execution_count": 27, "id": "e1350695-0f01-4205-a604-91d62896783e", "metadata": {}, "outputs": [ @@ -780,8 +788,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "Removing gs://essentials-student31-2021-10-26/...\n", - "NotEmptyException: 409 BucketNotEmpty (essentials-student31-2021-10-26)\n" + "Removing gs://essentials-learner-2022-01-14/...\n", + "NotEmptyException: 409 BucketNotEmpty (essentials-learner-2022-01-14)\n" ] } ], @@ -800,7 +808,7 @@ }, { "cell_type": "code", - "execution_count": 26, + "execution_count": 28, "id": "ba2671f8-521f-48b4-9231-491d128ba67c", "metadata": {}, "outputs": [ @@ -808,7 +816,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Removing gs://essentials-student31-2021-10-26/2...\n", + "Removing gs://essentials-learner-2022-01-14/2...\n", "/ [1 objects] \n", "Operation completed over 1 objects. \n" ] @@ -818,20 +826,20 @@ "gsutil rm \"gs://$BUCKET/2\"" ] }, - { - "cell_type": "markdown", - "id": "9540ba16-1662-49b5-9bf7-87c5c49b1e2c", - "metadata": {}, - "source": [ - "Finally we can remove the bucket. We first double check the value of `$BUCKET`" - ] - }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 29, "id": "73d8b299-8e8e-443e-ae5b-4ecf7efa0442", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "gs://essentials-learner-2022-01-14/\n" + ] + } + ], "source": [ "gsutil ls" ] @@ -847,7 +855,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 30, "id": "7b32764d-e7b5-42d6-8f88-bc6936b2024a", "metadata": {}, "outputs": [ @@ -855,7 +863,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "gs://essentials-student31-2021-10-26/\n" + "gs://essentials-learner-2022-01-14/\n" ] } ], @@ -873,7 +881,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 31, "id": "82487bd8-14f4-4811-8f5a-dd346042bd0f", "metadata": {}, "outputs": [ @@ -881,7 +889,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Bucket: essentials-student31-2021-10-26\n" + "Bucket: essentials-learner-2022-01-14\n" ] } ], @@ -891,7 +899,7 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 32, "id": "8635db94-2f3c-46d2-acc2-76a176fb37e7", "metadata": {}, "outputs": [ @@ -899,7 +907,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Removing gs://essentials-student31-2021-10-26/...\n" + "Removing gs://essentials-learner-2022-01-14/...\n" ] } ], @@ -917,7 +925,7 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 33, "id": "ed6f0204-0de2-4d20-ab00-3b7a1023fd86", "metadata": {}, "outputs": [],