diff --git a/content/GCP/05_cli_storage.ipynb b/content/GCP/05_cli_storage.ipynb index 00d761a..878850e 100644 --- a/content/GCP/05_cli_storage.ipynb +++ b/content/GCP/05_cli_storage.ipynb @@ -53,7 +53,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "998995103712-compute@developer.gserviceaccount.com\n" + "Your active configuration is: [cloudshell-27767]\n", + "learner@class.internet2.edu\n" ] } ], @@ -71,6 +72,7 @@ "name": "stdout", "output_type": "stream", "text": [ + "Your active configuration is: [cloudshell-27767]\n", "essentials-learner\n" ] } @@ -82,6 +84,25 @@ { "cell_type": "code", "execution_count": 3, + "id": "44f0b255-508a-4488-abde-87aae67cd5b3", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Your active configuration is: [cloudshell-27767]\n", + "learner@class.internet2.edu\n" + ] + } + ], + "source": [ + "ACCOUNT=$(gcloud config get-value account) ; echo $ACCOUNT" + ] + }, + { + "cell_type": "code", + "execution_count": 4, "id": "36fcc48b-8a5b-4158-9704-f1ea7c9951eb", "metadata": {}, "outputs": [ @@ -89,13 +110,13 @@ "name": "stdout", "output_type": "stream", "text": [ + "Your active configuration is: [cloudshell-27767]\n", "essentials-learner\n" ] } ], "source": [ - "PROJECT=$(gcloud config list --format='value(core.project)')\n", - "echo $PROJECT" + "PROJECT=$(gcloud config get-value project) ; echo $PROJECT" ] }, { @@ -118,7 +139,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 5, "id": "255eb914-a06a-4bfd-adbb-7ca2f94678e7", "metadata": {}, "outputs": [ @@ -126,18 +147,18 @@ "name": "stdout", "output_type": "stream", "text": [ - "essentials-learner-2022-01-21\n" + "essentials-learner-2022-02-03\n" ] } ], "source": [ - "BUCKET=\"essentials-${USER}-$(date +%F)\"\n", + "BUCKET=\"essentials-$USER-$(date +%F)\"\n", "echo $BUCKET" ] }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 6, "id": "b0661b4c-b56b-4d6d-ba3c-e1c5f38699c6", "metadata": {}, "outputs": [ @@ -145,12 +166,12 @@ "name": "stdout", "output_type": "stream", "text": [ - "Creating gs://essentials-learner-2022-01-21/...\n" + "Creating gs://essentials-learner-2022-02-03/...\n" ] } ], "source": [ - "gsutil mb \"gs://$BUCKET\"" + "gsutil mb gs://$BUCKET" ] }, { @@ -175,7 +196,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 7, "id": "8ae2988a-fca8-4601-a7e5-458627fe68e3", "metadata": {}, "outputs": [ @@ -183,7 +204,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "gs://essentials-learner-2022-01-21/\n" + "gs://essentials-learner-2022-02-03/\n" ] } ], @@ -209,7 +230,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 8, "id": "ecc0e4fe-c9d0-4edd-9a0c-56b5b9ec7c54", "metadata": { "scrolled": true, @@ -220,7 +241,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "gs://essentials-learner-2022-01-21/ :\n", + "gs://essentials-learner-2022-02-03/ :\n", "\tStorage class:\t\t\tSTANDARD\n", "\tLocation type:\t\t\tmulti-region\n", "\tLocation constraint:\t\tUS\n", @@ -232,71 +253,17 @@ "\tRequester Pays enabled:\t\tNone\n", "\tLabels:\t\t\t\tNone\n", "\tDefault KMS key:\t\tNone\n", - "\tTime created:\t\t\tFri, 21 Jan 2022 23:23:39 GMT\n", - "\tTime updated:\t\t\tFri, 21 Jan 2022 23:23:39 GMT\n", + "\tTime created:\t\t\tThu, 03 Feb 2022 22:19:56 GMT\n", + "\tTime updated:\t\t\tThu, 03 Feb 2022 22:19:56 GMT\n", "\tMetageneration:\t\t\t1\n", "\tBucket Policy Only enabled:\tFalse\n", "\tPublic access prevention:\tinherited\n", - "\tRPO:\t\t\t\tDEFAULT\n", - "\tACL:\t\t\t\t\n", - "\t [\n", - "\t {\n", - "\t \"entity\": \"project-owners-998995103712\",\n", - "\t \"projectTeam\": {\n", - "\t \"projectNumber\": \"998995103712\",\n", - "\t \"team\": \"owners\"\n", - "\t },\n", - "\t \"role\": \"OWNER\"\n", - "\t },\n", - "\t {\n", - "\t \"entity\": \"project-editors-998995103712\",\n", - "\t \"projectTeam\": {\n", - "\t \"projectNumber\": \"998995103712\",\n", - "\t \"team\": \"editors\"\n", - "\t },\n", - "\t \"role\": \"OWNER\"\n", - "\t },\n", - "\t {\n", - "\t \"entity\": \"project-viewers-998995103712\",\n", - "\t \"projectTeam\": {\n", - "\t \"projectNumber\": \"998995103712\",\n", - "\t \"team\": \"viewers\"\n", - "\t },\n", - "\t \"role\": \"READER\"\n", - "\t }\n", - "\t ]\n", - "\tDefault ACL:\t\t\t\n", - "\t [\n", - "\t {\n", - "\t \"entity\": \"project-owners-998995103712\",\n", - "\t \"projectTeam\": {\n", - "\t \"projectNumber\": \"998995103712\",\n", - "\t \"team\": \"owners\"\n", - "\t },\n", - "\t \"role\": \"OWNER\"\n", - "\t },\n", - "\t {\n", - "\t \"entity\": \"project-editors-998995103712\",\n", - "\t \"projectTeam\": {\n", - "\t \"projectNumber\": \"998995103712\",\n", - "\t \"team\": \"editors\"\n", - "\t },\n", - "\t \"role\": \"OWNER\"\n", - "\t },\n", - "\t {\n", - "\t \"entity\": \"project-viewers-998995103712\",\n", - "\t \"projectTeam\": {\n", - "\t \"projectNumber\": \"998995103712\",\n", - "\t \"team\": \"viewers\"\n", - "\t },\n", - "\t \"role\": \"READER\"\n", - "\t }\n", - "\t ]\n" + "\tRPO:\t\t\t\tDEFAULT\n" ] } ], "source": [ - "gsutil ls -L -b \"gs://$BUCKET\"" + "gsutil ls -L -b gs://$BUCKET | head -18" ] }, { @@ -306,14 +273,14 @@ "source": [ "You can see that the bucket is \"multi-region\" and uses the \"standard\" storage class (this is the type of storage). The standard storage class is best used for frequently used buckets. Other storage classes are designed for less frequent use, are less expensive, but come with more restrictions and more complex access costs and should only be used after careful consideration.\n", "\n", - "Ignore the \"ACL\" section for now.\n", + "*We used the `| head -18 ` command here to limit the response to the first few lines (we ignore ACL for now)*\n", "\n", "You may also want to verify that you can see the newly created bucket in the web console dashboard or \"Cloud Storage\" page and explore the properties there." ] }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 9, "id": "8fc85600-a793-413b-b139-642e4cf08a8e", "metadata": {}, "outputs": [], @@ -334,7 +301,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 10, "id": "bb8e7270-2f64-4fbd-b312-b18a7960700f", "metadata": { "scrolled": true, @@ -346,76 +313,20 @@ "output_type": "stream", "text": [ "---\n", - "insertId: -o4omdgeg7sge\n", + "insertId: -oywycvf1xe8xy\n", "logName: projects/essentials-learner/logs/cloudaudit.googleapis.com%2Factivity\n", "protoPayload:\n", " '@type': type.googleapis.com/google.cloud.audit.AuditLog\n", " authenticationInfo:\n", - " principalEmail: 998995103712-compute@developer.gserviceaccount.com\n", - " serviceAccountDelegationInfo:\n", - " - firstPartyPrincipal:\n", - " principalEmail: service-998995103712@compute-system.iam.gserviceaccount.com\n", + " principalEmail: learner@class.internet2.edu\n", " authorizationInfo:\n", " - granted: true\n", - " permission: storage.buckets.create\n", - " resource: projects/_/buckets/essentials-learner-2022-01-21\n", - " resourceAttributes: {}\n", - " metadata:\n", - " rpo: DEFAULT\n", - " methodName: storage.buckets.create\n", - " request:\n", - " defaultObjectAcl:\n", - " '@type': type.googleapis.com/google.iam.v1.Policy\n", - " bindings:\n", - " - members:\n", - " - projectViewer:essentials-learner\n", - " role: roles/storage.legacyObjectReader\n", - " - members:\n", - " - projectOwner:essentials-learner\n", - " - projectEditor:essentials-learner\n", - " role: roles/storage.legacyObjectOwner\n", - " requestMetadata:\n", - " callerIp: 34.102.126.22\n", - " callerNetwork: //compute.googleapis.com/projects/essentials-learner/global/networks/__unknown__\n", - " callerSuppliedUserAgent: apitools Python/3.7.3 gsutil/5.5 (linux) analytics/disabled\n", - " interactive/True command/mb google-cloud-sdk/366.0.0,gzip(gfe)\n", - " destinationAttributes: {}\n", - " requestAttributes:\n", - " auth: {}\n", - " time: '2022-01-21T23:23:39.060742076Z'\n", - " resourceLocation:\n", - " currentLocations:\n", - " - us\n", - " resourceName: projects/_/buckets/essentials-learner-2022-01-21\n", - " serviceData:\n", - " '@type': type.googleapis.com/google.iam.v1.logging.AuditData\n", - " policyDelta:\n", - " bindingDeltas:\n", - " - action: ADD\n", - " member: projectEditor:essentials-learner\n", - " role: roles/storage.legacyBucketOwner\n", - " - action: ADD\n", - " member: projectOwner:essentials-learner\n", - " role: roles/storage.legacyBucketOwner\n", - " - action: ADD\n", - " member: projectViewer:essentials-learner\n", - " role: roles/storage.legacyBucketReader\n", - " serviceName: storage.googleapis.com\n", - " status: {}\n", - "receiveTimestamp: '2022-01-21T23:23:39.729940907Z'\n", - "resource:\n", - " labels:\n", - " bucket_name: essentials-learner-2022-01-21\n", - " location: us\n", - " project_id: essentials-learner\n", - " type: gcs_bucket\n", - "severity: NOTICE\n", - "timestamp: '2022-01-21T23:23:39.055676566Z'\n" + " permission: storage.buckets.create\n" ] } ], "source": [ - "gcloud logging read --limit 1" + "gcloud logging read --limit 1 | head" ] }, { @@ -455,7 +366,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 11, "id": "f9a90752-871f-4156-b122-4c46fd4bfa43", "metadata": {}, "outputs": [], @@ -473,7 +384,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 12, "id": "bc0fb778-cc2e-4af4-8aa8-ca90d6b4f877", "metadata": {}, "outputs": [ @@ -499,7 +410,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 13, "id": "6384974b-d422-4ac1-90b7-cf9feec9aa8c", "metadata": {}, "outputs": [ @@ -514,12 +425,12 @@ } ], "source": [ - "gsutil cp one.txt \"gs://$BUCKET/1\"" + "gsutil cp one.txt gs://$BUCKET/1" ] }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 14, "id": "49589119-56bb-4c7b-b0a6-36e8f1133056", "metadata": {}, "outputs": [ @@ -527,12 +438,12 @@ "name": "stdout", "output_type": "stream", "text": [ - "gs://essentials-learner-2022-01-21/1\n" + "gs://essentials-learner-2022-02-03/1\n" ] } ], "source": [ - "gsutil ls \"gs://$BUCKET\"" + "gsutil ls gs://$BUCKET" ] }, { @@ -543,6 +454,24 @@ "Copy the object to the standard out (displays it on the screen)." ] }, + { + "cell_type": "code", + "execution_count": 15, + "id": "79d0f1ac-e42f-46c4-aeec-6548a2651015", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "test one\n" + ] + } + ], + "source": [ + "gsutil cat gs://$BUCKET/1" + ] + }, { "cell_type": "markdown", "id": "ef366240-1152-4b30-b99f-b7835f63788d", @@ -553,7 +482,7 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 16, "id": "442cc51f-8856-41d4-a3a6-072c8569beef", "metadata": {}, "outputs": [ @@ -571,7 +500,7 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 17, "id": "4e3c0de9-8a23-428a-9689-0cf896228afe", "metadata": {}, "outputs": [ @@ -584,7 +513,7 @@ } ], "source": [ - "gsutil cat \"gs://$BUCKET/1\"" + "gsutil cat gs://$BUCKET/1" ] }, { @@ -597,7 +526,7 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 18, "id": "13f94b36-7dea-46b9-9e8b-baa8b4e59de6", "metadata": {}, "outputs": [ @@ -605,19 +534,19 @@ "name": "stdout", "output_type": "stream", "text": [ - "Copying gs://essentials-learner-2022-01-21/1...\n", + "Copying gs://essentials-learner-2022-02-03/1...\n", "/ [1 files][ 9.0 B/ 9.0 B] \n", "Operation completed over 1 objects/9.0 B. \n" ] } ], "source": [ - "gsutil cp \"gs://$BUCKET/1\" one.txt" + "gsutil cp gs://$BUCKET/1 one.txt" ] }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 19, "id": "3982f21c-b827-4772-aa8c-3ac67e478b0d", "metadata": {}, "outputs": [ @@ -643,7 +572,7 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 20, "id": "6b57a3fc-94ea-4c25-b7ac-2744d7b28852", "metadata": {}, "outputs": [ @@ -669,7 +598,7 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 21, "id": "5a7fc453-fca7-4ab4-8f0a-cb4de53bc992", "metadata": {}, "outputs": [ @@ -684,7 +613,7 @@ } ], "source": [ - "date | gsutil cp - \"gs://$BUCKET/2\"" + "date | gsutil cp - gs://$BUCKET/2" ] }, { @@ -697,7 +626,7 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 22, "id": "1be58d86-5de7-420c-a3b3-0b82839ac90f", "metadata": {}, "outputs": [ @@ -705,8 +634,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "gs://essentials-learner-2022-01-21/1\n", - "gs://essentials-learner-2022-01-21/2\n" + "gs://essentials-learner-2022-02-03/1\n", + "gs://essentials-learner-2022-02-03/2\n" ] } ], @@ -721,13 +650,13 @@ "source": [ "```{admonition} Exercise\n", "\n", - "* Display the date in the bucket.\n", + "* Display the object with the date in the bucket.\n", "```" ] }, { "cell_type": "code", - "execution_count": 21, + "execution_count": 23, "id": "e3d0585b-4d81-4429-ad90-b3ff51eec13d", "metadata": {}, "outputs": [ @@ -735,12 +664,12 @@ "name": "stdout", "output_type": "stream", "text": [ - "Fri Jan 21 23:23:53 UTC 2022\n" + "Thu 03 Feb 2022 10:20:26 PM UTC\n" ] } ], "source": [ - "gsutil cat \"gs://$BUCKET/2\"" + "gsutil cat gs://$BUCKET/2" ] }, { @@ -753,7 +682,7 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": 24, "id": "9ad460d9-7a4f-4e3f-befb-2bbb10c51993", "metadata": {}, "outputs": [ @@ -761,19 +690,19 @@ "name": "stdout", "output_type": "stream", "text": [ - "Removing gs://essentials-learner-2022-01-21/1...\n", + "Removing gs://essentials-learner-2022-02-03/1...\n", "/ [1 objects] \n", "Operation completed over 1 objects. \n" ] } ], "source": [ - "gsutil rm \"gs://$BUCKET/1\"" + "gsutil rm gs://$BUCKET/1" ] }, { "cell_type": "code", - "execution_count": 23, + "execution_count": 25, "id": "8fd782f2-700a-4862-970b-1b4751009d5f", "metadata": {}, "outputs": [ @@ -781,12 +710,12 @@ "name": "stdout", "output_type": "stream", "text": [ - "gs://essentials-learner-2022-01-21/2\n" + "gs://essentials-learner-2022-02-03/2\n" ] } ], "source": [ - "gsutil ls \"gs://$BUCKET\"" + "gsutil ls gs://$BUCKET" ] }, { @@ -799,7 +728,7 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": 26, "id": "e1350695-0f01-4205-a604-91d62896783e", "metadata": {}, "outputs": [ @@ -807,13 +736,13 @@ "name": "stdout", "output_type": "stream", "text": [ - "Removing gs://essentials-learner-2022-01-21/...\n", - "NotEmptyException: 409 BucketNotEmpty (essentials-learner-2022-01-21)\n" + "Removing gs://essentials-learner-2022-02-03/...\n", + "NotEmptyException: 409 BucketNotEmpty (essentials-learner-2022-02-03)\n" ] } ], "source": [ - "gsutil rb \"gs://$BUCKET\"\n", + "gsutil rb gs://$BUCKET\n", "/bin/true # ignore expected error in Jupyter" ] }, @@ -827,7 +756,7 @@ }, { "cell_type": "code", - "execution_count": 25, + "execution_count": 27, "id": "ba2671f8-521f-48b4-9231-491d128ba67c", "metadata": {}, "outputs": [ @@ -835,19 +764,19 @@ "name": "stdout", "output_type": "stream", "text": [ - "Removing gs://essentials-learner-2022-01-21/2...\n", + "Removing gs://essentials-learner-2022-02-03/2...\n", "/ [1 objects] \n", "Operation completed over 1 objects. \n" ] } ], "source": [ - "gsutil rm \"gs://$BUCKET/2\"" + "gsutil rm gs://$BUCKET/2" ] }, { "cell_type": "code", - "execution_count": 26, + "execution_count": 28, "id": "73d8b299-8e8e-443e-ae5b-4ecf7efa0442", "metadata": {}, "outputs": [ @@ -855,7 +784,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "gs://essentials-learner-2022-01-21/\n" + "gs://essentials-learner-2022-02-03/\n" ] } ], @@ -874,7 +803,7 @@ }, { "cell_type": "code", - "execution_count": 27, + "execution_count": 29, "id": "7b32764d-e7b5-42d6-8f88-bc6936b2024a", "metadata": {}, "outputs": [ @@ -882,7 +811,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "gs://essentials-learner-2022-01-21/\n" + "gs://essentials-learner-2022-02-03/\n" ] } ], @@ -900,7 +829,7 @@ }, { "cell_type": "code", - "execution_count": 28, + "execution_count": 30, "id": "82487bd8-14f4-4811-8f5a-dd346042bd0f", "metadata": {}, "outputs": [ @@ -908,7 +837,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Bucket: essentials-learner-2022-01-21\n" + "Bucket: essentials-learner-2022-02-03\n" ] } ], @@ -916,9 +845,35 @@ "echo \"Bucket: $BUCKET\"" ] }, + { + "cell_type": "markdown", + "id": "6120eee3-47d0-4e28-a1dd-b1090297a573", + "metadata": {}, + "source": [ + "Check to see if the bucket is empty" + ] + }, { "cell_type": "code", - "execution_count": 29, + "execution_count": 31, + "id": "a52e1838-02f4-406c-a58f-6a709d9aaf2e", + "metadata": {}, + "outputs": [], + "source": [ + "gsutil ls gs://$BUCKET" + ] + }, + { + "cell_type": "markdown", + "id": "6eb97c7e-aed6-493a-b724-a4d6ba3cb806", + "metadata": {}, + "source": [ + "Remove the bucket" + ] + }, + { + "cell_type": "code", + "execution_count": 32, "id": "8635db94-2f3c-46d2-acc2-76a176fb37e7", "metadata": {}, "outputs": [ @@ -926,12 +881,12 @@ "name": "stdout", "output_type": "stream", "text": [ - "Removing gs://essentials-learner-2022-01-21/...\n" + "Removing gs://essentials-learner-2022-02-03/...\n" ] } ], "source": [ - "gsutil rb \"gs://$BUCKET\"" + "gsutil rb gs://$BUCKET" ] }, { @@ -939,12 +894,12 @@ "id": "3e2178ca", "metadata": {}, "source": [ - "We verify that the bucket has been removed. In this example there is no output since there are no more buckets." + "We verify that the bucket has been removed." ] }, { "cell_type": "code", - "execution_count": 30, + "execution_count": 33, "id": "ed6f0204-0de2-4d20-ab00-3b7a1023fd86", "metadata": {}, "outputs": [],